How to Index Fields for Vector Search

1
{
2
    "database": "sample_mflix",
3
    "collectionName": "embedded_movies",
4
    "type": "vectorSearch",
5
    "name": "vector_index",
6
    "fields": [ 
7
      {
8
        "type": "vector",
9
        "path": "plot_embedding",
10
        "numDimensions": 1536,
11
        "similarity": "dotProduct"
12
      }
13
    ]
14
}

1
{
2
    "database": "sample_mflix",
3
    "collectionName": "embedded_movies",
4
    "type": "vectorSearch",
5
    "name": "vector_index",
6
    "fields":[ 
7
      {
8
        "type": "vector",
9
        "path": "plot_embedding",
10
        "numDimensions": 1536,
11
        "similarity": "dotProduct"
12
      },
13
      {
14
        "type": "filter",
15
        "path": "genres"
16
      },
17
      {
18
        "type": "filter",
19
        "path": "year"
20
      }
21
    ]
22
}

1
{
2
  "fields": [{
3
    "type": "vector",
4
    "path": "plot_embedding",
5
    "numDimensions": 1536,
6
    "similarity": "dotProduct"
7
  }]
8
}

1
{
2
  "fields": [{
3
    "type": "vector",
4
    "path": "plot_embedding",
5
    "numDimensions": 1536,
6
    "similarity": "dotProduct"
7
  },
8
  {
9
    "type": "filter",
10
    "path": "genres"
11
  },
12
  {
13
    "type": "filter",
14
    "path": "year"
15
  }]
16
}

1
db.embedded_movies.createSearchIndex(
2
  "vector_index", 
3
  "vectorSearch", 
4
  {
5
    "fields": [
6
      {
7
        "type": "vector",
8
        "path": "plot_embedding",
9
        "numDimensions": 1536,
10
        "similarity": "dotProduct",
11
        "quantization": "scalar"
12
      }
13
    ]
14
  }
15
);

1
db.embedded_movies.createSearchIndex(
2
  "vector_index", 
3
  "vectorSearch", 
4
  {
5
    "fields": [
6
      {
7
        "type": "vector",
8
        "path": "plot_embedding",
9
        "numDimensions": 1536,
10
        "similarity": "dotProduct"
11
      },
12
      {
13
        "type": "filter",
14
        "path": "genres"
15
      },
16
      {
17
        "type": "filter",
18
        "path": "year"
19
      }
20
    ]
21
  }
22
);

1
namespace query_quick_start;
2
3
using MongoDB.Bson;
4
using MongoDB.Driver;
5
using System;
6
using System.Threading;
7
8
public class IndexService
9
{
10
    // Replace the placeholder with your Atlas connection string
11
    private const string MongoConnectionString = "<connection-string>";
12
    public void CreateVectorIndex()
13
    {
14
        try
15
        {
16
            // Connect to your Atlas cluster
17
            var client = new MongoClient(MongoConnectionString);
18
            var database = client.GetDatabase("sample_mflix");
19
            var collection = database.GetCollection<BsonDocument>("embedded_movies");
20
21
            var searchIndexView = collection.SearchIndexes;
22
            var name = "vector_index";
23
            var type = SearchIndexType.VectorSearch;
24
25
            var definition = new BsonDocument
26
            {
27
                { "fields", new BsonArray
28
                    {
29
                        new BsonDocument
30
                        {
31
                            { "type", "vector" },
32
                            { "path", "plot_embedding" },
33
                            { "numDimensions", 1536 },
34
                            { "similarity", "dotProduct" },
35
                            { "quantization", "scalar" }
36
                        }
37
                    }
38
                }
39
            };
40
41
            var model = new CreateSearchIndexModel(name, type, definition);
42
43
            searchIndexView.CreateOne(model);
44
            Console.WriteLine($"New search index named {name} is building.");
45
46
            // Polling for index status
47
            Console.WriteLine("Polling to check if the index is ready. This may take up to a minute.");
48
            bool queryable = false;
49
            while (!queryable)
50
            {
51
                var indexes = searchIndexView.List();
52
                foreach (var index in indexes.ToEnumerable())
53
                {
54
                    if (index["name"] == name)
55
                    {
56
                        queryable = index["queryable"].AsBoolean;
57
                    }
58
                }
59
                if (!queryable)
60
                {
61
                    Thread.Sleep(5000);
62
                }
63
            }
64
            Console.WriteLine($"{name} is ready for querying.");
65
        }
66
        catch (Exception e)
67
        {
68
            Console.WriteLine($"Exception: {e.Message}");
69
        }
70
    }
71
}

1
namespace query_quick_start;
2
3
using MongoDB.Bson;
4
using MongoDB.Driver;
5
using System;
6
using System.Threading;
7
8
public class IndexService
9
{
10
    // Replace the placeholder with your Atlas connection string
11
    private const string MongoConnectionString = "<connection-string>";
12
    public void CreateVectorIndex()
13
    {
14
        try
15
        {
16
            // Connect to your Atlas cluster
17
            var client = new MongoClient(MongoConnectionString);
18
            var database = client.GetDatabase("sample_mflix");
19
            var collection = database.GetCollection<BsonDocument>("embedded_movies");
20
21
            var searchIndexView = collection.SearchIndexes;
22
            var name = "vector_index";
23
            var type = SearchIndexType.VectorSearch;
24
25
            var definition = new BsonDocument
26
            {
27
                { "fields", new BsonArray
28
                    {
29
                        new BsonDocument
30
                        {
31
                            { "type", "vector" },
32
                            { "path", "plot_embedding" },
33
                            { "numDimensions", 1536 },
34
                            { "similarity", "dotProduct" },
35
                            { "quantization", "scalar"}
36
                        },
37
                        new BsonDocument
38
                        {
39
                            {"type", "filter"},
40
                            {"path", "genres"}
41
                        },
42
                        new BsonDocument
43
                        {
44
                            {"type", "filter"},
45
                            {"path", "year"}
46
                        }
47
                    }
48
                }
49
            };
50
51
            var model = new CreateSearchIndexModel(name, type, definition);
52
53
            searchIndexView.CreateOne(model);
54
            Console.WriteLine($"New search index named {name} is building.");
55
56
            // Polling for index status
57
            Console.WriteLine("Polling to check if the index is ready. This may take up to a minute.");
58
            bool queryable = false;
59
            while (!queryable)
60
            {
61
                var indexes = searchIndexView.List();
62
                foreach (var index in indexes.ToEnumerable())
63
                {
64
                    if (index["name"] == name)
65
                    {
66
                        queryable = index["queryable"].AsBoolean;
67
                    }
68
                }
69
                if (!queryable)
70
                {
71
                    Thread.Sleep(5000);
72
                }
73
            }
74
            Console.WriteLine($"{name} is ready for querying.");
75
        }
76
        catch (Exception e)
77
        {
78
            Console.WriteLine($"Exception: {e.Message}");
79
        }
80
    }
81
}

1
package main
2
3
import (
4
	"context"
5
	"fmt"
6
	"log"
7
	"time"
8
9
	"go.mongodb.org/mongo-driver/bson"
10
	"go.mongodb.org/mongo-driver/mongo"
11
	"go.mongodb.org/mongo-driver/mongo/options"
12
)
13
14
func main() {
15
	ctx := context.Background()
16
17
	// Replace the placeholder with your Atlas connection string
18
	const uri = "<connectionString>"
19
20
	// Connect to your Atlas cluster
21
	clientOptions := options.Client().ApplyURI(uri)
22
	client, err := mongo.Connect(ctx, clientOptions)
23
	if err != nil {
24
		log.Fatalf("failed to connect to the server: %v", err)
25
	}
26
	defer func() { _ = client.Disconnect(ctx) }()
27
28
	// Set the namespace
29
	coll := client.Database("sample_mflix").Collection("embedded_movies")
30
31
	// Define the index details
32
	type vectorDefinitionField struct {
33
		Type          string `bson:"type"`
34
		Path          string `bson:"path"`
35
		NumDimensions int    `bson:"numDimensions"`
36
		Similarity    string `bson:"similarity"`
37
		Quantization  string `bson:"quantization"`
38
	}
39
40
	type vectorDefinition struct {
41
		Fields []vectorDefinitionField `bson:"fields"`
42
	}
43
44
	indexName := "vector_index"
45
	opts := options.SearchIndexes().SetName(indexName).SetType("vectorSearch")
46
47
	indexModel := mongo.SearchIndexModel{
48
		Definition: vectorDefinition{
49
			Fields: []vectorDefinitionField{{
50
				Type:          "vector",
51
				Path:          "plot_embedding",
52
				NumDimensions: 1536,
53
				Similarity:    "dotProduct",
54
			    Quantization:  "scalar"}},
55
		},
56
		Options: opts,
57
	}
58
59
	// Create the index
60
	searchIndexName, err := coll.SearchIndexes().CreateOne(ctx, indexModel)
61
	if err != nil {
62
		log.Fatalf("failed to create the search index: %v", err)
63
	}
64
	log.Println("New search index named " + searchIndexName + " is building.")
65
66
	// Await the creation of the index.
67
	log.Println("Polling to check if the index is ready. This may take up to a minute.")
68
	searchIndexes := coll.SearchIndexes()
69
	var doc bson.Raw
70
	for doc == nil {
71
		cursor, err := searchIndexes.List(ctx, options.SearchIndexes().SetName(searchIndexName))
72
		if err != nil {
73
			fmt.Errorf("failed to list search indexes: %w", err)
74
		}
75
76
		if !cursor.Next(ctx) {
77
			break
78
		}
79
80
		name := cursor.Current.Lookup("name").StringValue()
81
		queryable := cursor.Current.Lookup("queryable").Boolean()
82
		if name == searchIndexName && queryable {
83
			doc = cursor.Current
84
		} else {
85
			time.Sleep(5 * time.Second)
86
		}
87
	}
88
89
	log.Println(searchIndexName + " is ready for querying.")
90
}

1
package main
2
3
import (
4
	"context"
5
	"fmt"
6
	"log"
7
	"time"
8
9
	"go.mongodb.org/mongo-driver/bson"
10
	"go.mongodb.org/mongo-driver/mongo"
11
	"go.mongodb.org/mongo-driver/mongo/options"
12
)
13
14
func main() {
15
	ctx := context.Background()
16
17
	// Replace the placeholder with your Atlas connection string
18
	const uri = "<connectionString>"
19
20
	// Connect to your Atlas cluster
21
	clientOptions := options.Client().ApplyURI(uri)
22
	client, err := mongo.Connect(ctx, clientOptions)
23
	if err != nil {
24
		log.Fatalf("failed to connect to the server: %v", err)
25
	}
26
	defer func() { _ = client.Disconnect(ctx) }()
27
28
	// Set the namespace
29
	coll := client.Database("sample_mflix").Collection("embedded_movies")
30
	indexName := "vector_index"
31
	opts := options.SearchIndexes().SetName(indexName).SetType("vectorSearch")
32
33
	type vectorDefinitionField struct {
34
		Type          string `bson:"type"`
35
		Path          string `bson:"path"`
36
		NumDimensions int    `bson:"numDimensions"`
37
		Similarity    string `bson:"similarity"`
38
		Quantization  string `bson:"quantization"`
39
	}
40
41
	type filterField struct {
42
		Type string `bson:"type"`
43
		Path string `bson:"path"`
44
	}
45
46
	type indexDefinition struct {
47
		Fields []vectorDefinitionField `bson:"fields"`
48
	}
49
50
	vectorDefinition := vectorDefinitionField{
51
		Type:          "vector",
52
		Path:          "plot_embedding",
53
		NumDimensions: 1536,
54
		Similarity:    "dotProduct",
55
	        Quantization:  "scalar"}
56
	genreFilterDefinition := filterField{"filter", "genres"}
57
	yearFilterDefinition := filterField{"filter", "year"}
58
59
	indexModel := mongo.SearchIndexModel{
60
		Definition: bson.D{{"fields", [3]interface{}{
61
			vectorDefinition,
62
			genreFilterDefinition,
63
			yearFilterDefinition}}},
64
		Options: opts,
65
	}
66
67
	// Create the index
68
	searchIndexName, err := coll.SearchIndexes().CreateOne(ctx, indexModel)
69
	if err != nil {
70
		log.Fatalf("failed to create the search index: %v", err)
71
	}
72
	log.Println("New search index named " + searchIndexName + " is building.")
73
74
	// Await the creation of the index.
75
	log.Println("Polling to check if the index is ready. This may take up to a minute.")
76
	searchIndexes := coll.SearchIndexes()
77
	var doc bson.Raw
78
	for doc == nil {
79
		cursor, err := searchIndexes.List(ctx, options.SearchIndexes().SetName(searchIndexName))
80
		if err != nil {
81
			fmt.Errorf("failed to list search indexes: %w", err)
82
		}
83
84
		if !cursor.Next(ctx) {
85
			break
86
		}
87
88
		name := cursor.Current.Lookup("name").StringValue()
89
		queryable := cursor.Current.Lookup("queryable").Boolean()
90
		if name == searchIndexName && queryable {
91
			doc = cursor.Current
92
		} else {
93
			time.Sleep(5 * time.Second)
94
		}
95
	}
96
97
	log.Println(searchIndexName + " is ready for querying.")
98
}

1
const { MongoClient } = require("mongodb");
2
3
// connect to your Atlas deployment
4
const uri =  "<connectionString>";
5
6
const client = new MongoClient(uri);
7
8
async function run() {
9
   try {
10
     const database = client.db("sample_mflix");
11
     const collection = database.collection("embedded_movies");
12
    
13
     // define your Atlas Vector Search index
14
     const index = {
15
         name: "vector_index",
16
         type: "vectorSearch",
17
         definition: {
18
           "fields": [
19
             {
20
               "type": "vector",
21
               "numDimensions": 1536,
22
               "path": "plot_embedding",
23
               "similarity": "dotProduct",
24
               "quantization": "scalar"
25
             }
26
           ]
27
         }
28
     }
29
30
     // run the helper method
31
     const result = await collection.createSearchIndex(index);
32
     console.log(`New search index named ${result} is building.`);
33
34
     // wait for the index to be ready to query
35
     console.log("Polling to check if the index is ready. This may take up to a minute.")
36
     let isQueryable = false;
37
     while (!isQueryable) {
38
       const cursor = collection.listSearchIndexes();
39
       for await (const index of cursor) {
40
         if (index.name === result) {
41
           if (index.queryable) {
42
             console.log(`${result} is ready for querying.`);
43
             isQueryable = true;
44
           } else {
45
             await new Promise(resolve => setTimeout(resolve, 5000));
46
           }
47
         }
48
       }
49
     }
50
   } finally {
51
     await client.close();
52
   }
53
}
54
run().catch(console.dir);

1
const { MongoClient } = require("mongodb");
2
3
// connect to your Atlas deployment
4
const uri =  "<connectionString>";
5
6
const client = new MongoClient(uri);
7
8
async function run() {
9
   try {
10
     const database = client.db("sample_mflix");
11
     const collection = database.collection("embedded_movies");
12
    
13
     // define your Atlas Vector Search index
14
     const index = {
15
         name: "vector_index",
16
         type: "vectorSearch",
17
         definition: {
18
           "fields": [
19
             {
20
               "type": "vector",
21
               "numDimensions": 1536,
22
               "path": "plot_embedding",
23
               "similarity": "dotProduct",
24
               "quantization": "scalar"
25
             },
26
             {
27
               "type": "filter",
28
               "path": "genres"
29
             },
30
             {
31
               "type": "filter",
32
               "path": "year"
33
             }
34
           ]
35
         }
36
     }
37
38
     // run the helper method
39
     const result = await collection.createSearchIndex(index);
40
     console.log(`New search index named ${result} is building.`);
41
42
     // wait for the index to be ready to query
43
     console.log("Polling to check if the index is ready. This may take up to a minute.")
44
     let isQueryable = false;
45
     while (!isQueryable) {
46
       const cursor = collection.listSearchIndexes();
47
       for await (const index of cursor) {
48
         if (index.name === result) {
49
           if (index.queryable) {
50
             console.log(`${result} is ready for querying.`);
51
             isQueryable = true;
52
           } else {
53
             await new Promise(resolve => setTimeout(resolve, 5000));
54
           }
55
         }
56
       }
57
     }
58
   } finally {
59
     await client.close();
60
   }
61
}
62
run().catch(console.dir);