sync-orama-cloud.mjs

import { OramaCloud } from '@orama/core';

import { getDocuments } from './get-documents.mjs';
import { ORAMA_SYNC_BATCH_SIZE } from '../../next.constants.mjs';

// The following follows the instructions at https://docs.oramasearch.com/docs/cloud/data-sources/rest-APIs/using-rest-apis

const orama = new OramaCloud({
  projectId: process.env.NEW_ORAMA_PROJECT_ID || '',
  apiKey: process.env.NEW_ORAMA_PRIVATE_API_KEY || '',
});

const datasource = orama.dataSource(process.env.NEW_ORAMA_DATASOURCE_ID || '');

// Create a temporary index to perform the insertions
const temporary = await datasource.createTemporaryIndex();
const documents = await getDocuments();

console.log(`Syncing ${documents.length} documents to Orama Cloud index`);

// Orama allows to send several documents at once, so we batch them in groups of 50.
// This is not strictly necessary, but it makes the process faster.
const runUpdate = async () => {
  const batchSize = ORAMA_SYNC_BATCH_SIZE;
  const batches = [];

  for (let i = 0; i < documents.length; i += batchSize) {
    batches.push(documents.slice(i, i + batchSize));
  }

  console.log(`Sending ${batches.length} batches of ${batchSize} documents`);

  // Insert documents batch by batch into the temporary index
  for (const batch of batches) {
    await temporary.insertDocuments(batch);
  }

  // Once all documents are inserted into the temporary index, we swap it with the live one atomically.
  await temporary.swap();
};

await runUpdate();

console.log('Orama Cloud sync completed successfully!');