Add SearchByMetadata method to SqlLiteVectorCollection

src/Abstractions/src/IVectorCollection.cs

@@ -57,6 +57,16 @@
         VectorSearchSettings? settings = default,
         CancellationToken cancellationToken = default);
 
+    /// <summary>
+    /// Searches for records based on a metadata filter from a specific collection.
+    /// </summary>
+    /// <param name="filters">The filters to apply to the search request.</param>
+    /// <param name="cancellationToken">The cancellation token.</param>
+    /// <returns>A task representing the asynchronous operation. The task result contains the search response.</returns>
+    Task<List<Vector>> SearchByMetadata(
+        Dictionary<string, object> filters,
+        CancellationToken cancellationToken = default);
+
     /// <summary>
     /// Checks if the collection is empty.
     /// </summary>

src/Chroma/src/ChromaVectorCollection.cs

@@ -166,6 +166,11 @@ private static IDictionary<string, object> DeserializeMetadata(MemoryRecordMetad
         return JsonSerializer.Deserialize(metadata.AdditionalMetadata, SourceGenerationContext.Default.IDictionaryStringObject)
                ?? new Dictionary<string, object>();
     }
+
+    Task<List<Vector>> IVectorCollection.SearchByMetadata(Dictionary<string, object> filters, CancellationToken cancellationToken)
+    {
+        throw new NotSupportedException("SemanticKernel doesn't support collection metadata");
+    }
 }
 
 [JsonSourceGenerationOptions(Converters = [typeof(ObjectAsPrimitiveConverter)])]

src/Elasticsearch/src/ElasticsearchVectorCollection.cs

@@ -103,4 +103,9 @@ public Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default)
     {
         throw new NotImplementedException();
     }
+
+    Task<List<Vector>> IVectorCollection.SearchByMetadata(Dictionary<string, object> filters, CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException();
+    }
 }

src/InMemory/src/InMemoryVectorCollection.cs

@@ -91,4 +91,9 @@ public Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default)
     {
         return Task.FromResult(_vectors.GetValueOrDefault(id));
     }
+
+    Task<List<Vector>> IVectorCollection.SearchByMetadata(Dictionary<string, object> filters, CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Mongo/src/MongoVectorCollection.cs

@@ -70,4 +70,9 @@ public async Task<VectorSearchResponse> SearchAsync(VectorSearchRequest request,
             .ToArray(),
         };
     }
+
+    Task<List<Vector>> IVectorCollection.SearchByMetadata(Dictionary<string, object> filters, CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException();
+    }
 }

src/OpenSearch/src/OpenSearchVectorCollection.cs

@@ -133,4 +133,9 @@ public Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default)
     {
         throw new NotImplementedException();
     }
+
+    Task<List<Vector>> IVectorCollection.SearchByMetadata(Dictionary<string, object> filters, CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Postgres/src/PostgresVectorCollection.cs

@@ -1,3 +1,4 @@
+
 namespace LangChain.Databases.Postgres;
 
 /// <summary>
@@ -123,4 +124,9 @@ public Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default)
     {
         throw new NotImplementedException();
     }
+
+    Task<List<Vector>> IVectorCollection.SearchByMetadata(Dictionary<string, object> filters, CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException();
+    }
 }

src/SemanticKernel/src/SemanticKernelMemoryStoreCollection.cs

@@ -75,4 +75,9 @@ public async Task<VectorSearchResponse> SearchAsync(VectorSearchRequest request,
             .ToListAsync(cancellationToken).ConfigureAwait(false);
         return new VectorSearchResponse { Items = results.Select(x => new Vector { Text = x.Item1.Metadata.ExternalSourceName }).ToList() };
     }
+
+    Task<List<Vector>> IVectorCollection.SearchByMetadata(Dictionary<string, object> filters, CancellationToken cancellationToken)
+    {
+        throw new NotImplementedException();
+    }
 }

src/Sqlite/src/SqLiteVectorCollection.cs

@@ -1,6 +1,7 @@
-using System.Globalization;
+using Microsoft.Data.Sqlite;
+using System.Globalization;
 using System.Text.Json;
-using Microsoft.Data.Sqlite;
+using System.Text.RegularExpressions;
 
 namespace LangChain.Databases.Sqlite;
 
@@ -37,49 +38,56 @@ private static string SerializeVector(float[] vector)
 
     private async Task InsertDocument(string id, float[] vector, Vector document)
     {
-        var insertCommand = _connection.CreateCommand();
-        string query = $"INSERT INTO {Name} (id, vector, document) VALUES (@id, @vector, @document)";
-        insertCommand.CommandText = query;
-        insertCommand.Parameters.AddWithValue("@id", id);
-        insertCommand.Parameters.AddWithValue("@vector", SerializeVector(vector));
-        insertCommand.Parameters.AddWithValue("@document", SerializeDocument(document));
-        await insertCommand.ExecuteNonQueryAsync().ConfigureAwait(false);
-
+        using (var insertCommand = _connection.CreateCommand())
+        {
+            string query = $"INSERT INTO {Name} (id, vector, document) VALUES (@id, @vector, @document)";
+            insertCommand.CommandText = query;
+            insertCommand.Parameters.AddWithValue("@id", id);
+            insertCommand.Parameters.AddWithValue("@vector", SerializeVector(vector));
+            insertCommand.Parameters.AddWithValue("@document", SerializeDocument(document));
+            await insertCommand.ExecuteNonQueryAsync().ConfigureAwait(false);
+        }
     }
 
     private async Task DeleteDocument(string id)
     {
-        var deleteCommand = _connection.CreateCommand();
-        string query = $"DELETE FROM {Name} WHERE id=@id";
-        deleteCommand.CommandText = query;
-        deleteCommand.Parameters.AddWithValue("@id", id);
-        await deleteCommand.ExecuteNonQueryAsync().ConfigureAwait(false);
+        using (var deleteCommand = _connection.CreateCommand())
+        {
+            string query = $"DELETE FROM {Name} WHERE id=@id";
+            deleteCommand.CommandText = query;
+            deleteCommand.Parameters.AddWithValue("@id", id);
+            await deleteCommand.ExecuteNonQueryAsync().ConfigureAwait(false);
+        }
     }
 
     private async Task<List<(Vector, float)>> SearchByVector(float[] vector, int k)
     {
-        var searchCommand = _connection.CreateCommand();
-        string query = $"SELECT id, vector, document, distance(vector, @vector) d FROM {Name} ORDER BY d LIMIT @k";
-        searchCommand.CommandText = query;
-        searchCommand.Parameters.AddWithValue("@vector", SerializeVector(vector));
-        searchCommand.Parameters.AddWithValue("@k", k);
-        var res = new List<(Vector, float)>();
-        var reader = await searchCommand.ExecuteReaderAsync().ConfigureAwait(false);
-        while (await reader.ReadAsync().ConfigureAwait(false))
+        using (var searchCommand = _connection.CreateCommand())
         {
-            var id = reader.GetString(0);
-            var vec = await reader.GetFieldValueAsync<string>(1).ConfigureAwait(false);
-            var doc = await reader.GetFieldValueAsync<string>(2).ConfigureAwait(false);
-            var docDeserialized = JsonSerializer.Deserialize(doc, SourceGenerationContext.Default.Vector) ?? new Vector
+            string query = $"SELECT id, vector, document, distance(vector, @vector) d FROM {Name} ORDER BY d LIMIT @k";
+            searchCommand.CommandText = query;
+            searchCommand.Parameters.AddWithValue("@vector", SerializeVector(vector));
+            searchCommand.Parameters.AddWithValue("@k", k);
+            var res = new List<(Vector, float)>();
+
+            using (var reader = await searchCommand.ExecuteReaderAsync().ConfigureAwait(false))
             {
-                Text = string.Empty,
-            };
-            var distance = reader.GetFloat(3);
-            res.Add((docDeserialized, distance));
+                while (await reader.ReadAsync().ConfigureAwait(false))
+                {
+                    var id = reader.GetString(0);
+                    var vec = await reader.GetFieldValueAsync<string>(1).ConfigureAwait(false);
+                    var doc = await reader.GetFieldValueAsync<string>(2).ConfigureAwait(false);
+                    var docDeserialized = JsonSerializer.Deserialize(doc, SourceGenerationContext.Default.Vector) ?? new Vector
+                    {
+                        Text = string.Empty,
+                    };
+                    var distance = reader.GetFloat(3);
+                    res.Add((docDeserialized, distance));
+                }
 
+                return res;
+            }
         }
-
-        return res;
     }
 
     /// <inheritdoc />
@@ -109,41 +117,48 @@ public async Task<IReadOnlyCollection<string>> AddAsync(
     /// <inheritdoc />
     public async Task<Vector?> GetAsync(string id, CancellationToken cancellationToken = default)
     {
-        var command = _connection.CreateCommand();
-        var query = $"SELECT vector, document FROM {Name} WHERE id=@id";
-        command.CommandText = query;
-        command.Parameters.AddWithValue("@id", id);
-        var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false);
-        if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+        using (var command = _connection.CreateCommand())
         {
-            return null;
-        }
+            var query = $"SELECT vector, document FROM {Name} WHERE id=@id";
+            command.CommandText = query;
+            command.Parameters.AddWithValue("@id", id);
 
-        var vec = await reader.GetFieldValueAsync<string>(0, cancellationToken).ConfigureAwait(false);
-        var doc = await reader.GetFieldValueAsync<string>(1, cancellationToken).ConfigureAwait(false);
-        var docDeserialized = JsonSerializer.Deserialize(doc, SourceGenerationContext.Default.Vector) ?? new Vector
-        {
-            Text = string.Empty,
-        };
+            using (var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false))
+            {
+                if (!await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+                {
+                    return null;
+                }
 
-        return new Vector
-        {
-            Id = id,
-            Text = docDeserialized.Text,
-            Metadata = docDeserialized.Metadata,
-            Embedding = JsonSerializer.Deserialize(vec, SourceGenerationContext.Default.SingleArray),
-        };
+                var vec = await reader.GetFieldValueAsync<string>(0, cancellationToken).ConfigureAwait(false);
+                var doc = await reader.GetFieldValueAsync<string>(1, cancellationToken).ConfigureAwait(false);
+                var docDeserialized = JsonSerializer.Deserialize(doc, SourceGenerationContext.Default.Vector) ?? new Vector
+                {
+                    Text = string.Empty,
+                };
+
+                return new Vector
+                {
+                    Id = id,
+                    Text = docDeserialized.Text,
+                    Metadata = docDeserialized.Metadata,
+                    Embedding = JsonSerializer.Deserialize(vec, SourceGenerationContext.Default.SingleArray),
+                };
+            }
+        }
     }
 
     /// <inheritdoc />
     public async Task<bool> IsEmptyAsync(CancellationToken cancellationToken = default)
     {
-        var command = _connection.CreateCommand();
-        var query = $"SELECT COUNT(*) FROM {Name}";
-        command.CommandText = query;
-        var count = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
+        using (var command = _connection.CreateCommand())
+        {
+            var query = $"SELECT COUNT(*) FROM {Name}";
+            command.CommandText = query;
+            var count = await command.ExecuteScalarAsync(cancellationToken).ConfigureAwait(false);
 
-        return count == null || Convert.ToInt32(count, CultureInfo.InvariantCulture) == 0;
+            return count == null || Convert.ToInt32(count, CultureInfo.InvariantCulture) == 0;
+        }
     }
 
     /// <inheritdoc />
@@ -180,4 +195,67 @@ public async Task<VectorSearchResponse> SearchAsync(
             }).ToArray(),
         };
     }
+
+    /// <inheritdoc />
+    public async Task<List<Vector>> SearchByMetadata(
+    Dictionary<string, object> filters,
+    CancellationToken cancellationToken = default)
+    {
+        filters = filters ?? throw new ArgumentNullException(nameof(filters));
+
+        using (var command = _connection.CreateCommand())
+        {
+            var query = $"SELECT id, vector, document FROM {Name}";
+
+            var whereClauses = new List<string>();
+            int paramIndex = 0;
+
+            foreach (var filter in filters)
+            {
+                // Validate filter.Key to prevent injection in JSON path
+                if (string.IsNullOrWhiteSpace(filter.Key) || !IsValidJsonKey(filter.Key))
+                    throw new ArgumentException($"Invalid filter key: {filter.Key}", nameof(filters));
+
+                var paramName = "@param" + paramIndex++;
+                whereClauses.Add($"json_extract(document, '$.Metadata.{filter.Key}') = {paramName}");
+                command.Parameters.AddWithValue(paramName, filter.Value);
+            }
+            query += " WHERE " + string.Join(" AND ", whereClauses);
+
+            command.CommandText = query;
+            var res = new List<Vector>();
+
+            using (var reader = await command.ExecuteReaderAsync(cancellationToken).ConfigureAwait(false))
+            {
+                while (await reader.ReadAsync(cancellationToken).ConfigureAwait(false))
+                {
+                    var id = await reader.GetFieldValueAsync<string>(0, cancellationToken).ConfigureAwait(false);
+                    var vec = await reader.GetFieldValueAsync<string>(1, cancellationToken).ConfigureAwait(false);
+                    var doc = await reader.GetFieldValueAsync<string>(2, cancellationToken).ConfigureAwait(false);
+                    var docDeserialized = JsonSerializer.Deserialize(doc, SourceGenerationContext.Default.Vector) ?? new Vector
+                    {
+                        Text = string.Empty,
+                    };
+
+                    var vector = new Vector
+                    {
+                        Id = id,
+                        Text = docDeserialized.Text,
+                        Metadata = docDeserialized.Metadata,
+                        Embedding = JsonSerializer.Deserialize(vec, SourceGenerationContext.Default.SingleArray),
+                    };
+
+                    res.Add(vector);
+                }
+
+                return res;
+            }
+        }
+    }
+
+    private static bool IsValidJsonKey(string input)
+    {
+        // Only allow letters, numbers, and underscores
+        return Regex.IsMatch(input, @"^\w+$");
+    }
 }