Migrate to a new embedding model for a collection (C#)

Follow this migration guide if you want to switch embedding models for your collection.

This migration only works for documents that have a $vectorize field.

Create a new collection with the desired embedding provider integration. For examples, see Create a collection that can automatically generate vector embeddings.

Migrate your documents to the new collection.

Exclude the $vector field from the migrated documents. The embedding provider integration for your new collection will automatically generate vector embeddings based on the $vectorize field and store them in the $vector field. Any documents without a $vectorize field will not have their $vector field automatically populated.

For example:

using DataStax.AstraDB.DataApi;
using DataStax.AstraDB.DataApi.Collections;
using DataStax.AstraDB.DataApi.Core;
using DataStax.AstraDB.DataApi.Core.Enumeration;
using DataStax.AstraDB.DataApi.Core.Query;

namespace Examples;

public class Program
{
  static async Task Main()
  {
    var client = new DataAPIClient();
    var database = client.GetDatabase(
      "API_ENDPOINT",
      "APPLICATION_TOKEN"
    );

    var oldCollection = database.GetCollection("OLD_COLLECTION_NAME");
    var newCollection = database.GetCollection("NEW_COLLECTION_NAME");

    string? pageState = null;
    int migratedCount = 0;

    // Use an empty filter to migrate all documents
    var filterBuilder = Builders<Document>.CollectionFilter;
    var filter = filterBuilder.Empty();

    // You must explicitly include $vectorize.
    // $vector is excluded by default.
    // _id and any other fields that don't start with $ are included by default.
    var projection = Builders<Document>.Projection.Include("$vectorize");

    while (true)
    {
      CollectionFindCursor<Document> cursor;
      if (pageState != null)
      {
        cursor = oldCollection.Find(
          filter,
          new CollectionFindOptions<Document>()
          {
            InitialPageState = pageState,
            Projection = projection,
          }
        );
      }
      else
      {
        cursor = oldCollection.Find(
          filter,
          new CollectionFindOptions<Document>()
          {
            Projection = projection,
          }
        );
      }

      var page = await cursor.FetchNextPageAsync();
      var documents = page.Results;
      pageState = page.NextPageState;

      if (documents.Count == 0)
      {
        Console.WriteLine("No more documents. Migration complete.");
        break;
      }

      // Insert the documents to the new collection.
      // _id and the other field values (excluding $vector) will be the same.
      // $vector will automatically be generated based on the value $vectorize.
      await newCollection.InsertManyAsync(documents);

      migratedCount += documents.Count;

      Console.WriteLine(
        $"Migrated {migratedCount} documents. Page state: {pageState}"
      );

      if (pageState == null)
      {
        Console.WriteLine("Reached final page. Migration complete.");
        break;
      }
    }
  }
}

Optionally, delete the collection that stores the old vector embeddings.

Migrate to a new embedding model for a collection (C#)

Was this helpful?

Give Feedback