Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a base document-index to allow indexing non-content items in AzureAI search #16611

Merged
merged 3 commits into from
Aug 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -206,7 +206,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe

switch (entry.Type)
{
case DocumentIndex.Types.Boolean:
case DocumentIndexBase.Types.Boolean:
// Store "true"/"false" for boolean.
doc.Add(new StringField(entry.Name, Convert.ToString(entry.Value).ToLowerInvariant(), store));

Expand All @@ -216,7 +216,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.DateTime:
case DocumentIndexBase.Types.DateTime:
if (entry.Value != null)
{
if (entry.Value is DateTimeOffset)
Expand Down Expand Up @@ -246,7 +246,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.Integer:
case DocumentIndexBase.Types.Integer:
if (entry.Value != null && long.TryParse(entry.Value.ToString(), out var value))
{
doc.Add(new Int64Field(entry.Name, value, store));
Expand All @@ -263,7 +263,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe

break;

case DocumentIndex.Types.Number:
case DocumentIndexBase.Types.Number:
if (entry.Value != null)
{
doc.Add(new DoubleField(entry.Name, Convert.ToDouble(entry.Value), store));
Expand All @@ -279,7 +279,7 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.Text:
case DocumentIndexBase.Types.Text:
if (entry.Value != null && !string.IsNullOrEmpty(Convert.ToString(entry.Value)))
{
var stringValue = Convert.ToString(entry.Value);
Expand Down Expand Up @@ -319,10 +319,10 @@ private Document CreateLuceneDocument(DocumentIndex documentIndex, LuceneIndexSe
}
break;

case DocumentIndex.Types.GeoPoint:
case DocumentIndexBase.Types.GeoPoint:
var strategy = new RecursivePrefixTreeStrategy(_grid, entry.Name);

if (entry.Value != null && entry.Value is DocumentIndex.GeoPoint point)
if (entry.Value != null && entry.Value is DocumentIndexBase.GeoPoint point)
{
var geoPoint = _ctx.MakePoint((double)point.Longitude, (double)point.Latitude);
foreach (var field in strategy.CreateIndexableFields(geoPoint))
Expand Down Expand Up @@ -422,7 +422,7 @@ private IndexReaderPool.IndexReaderLease GetReader(string indexName)
}

/// <summary>
/// Releases all readers and writers. This can be used after some time of innactivity to free resources.
/// Releases all readers and writers. This can be used after some time of inactivity to free resources.
/// </summary>
public void FreeReaderWriter()
{
Expand Down Expand Up @@ -456,7 +456,7 @@ public void FreeReaderWriter()
}

/// <summary>
/// Releases all readers and writers. This can be used after some time of innactivity to free resources.
/// Releases all readers and writers. This can be used after some time of inactivity to free resources.
/// </summary>
public void FreeReaderWriter(string indexName)
{
Expand Down
74 changes: 6 additions & 68 deletions src/OrchardCore/OrchardCore.Indexing.Abstractions/DocumentIndex.cs
Original file line number Diff line number Diff line change
@@ -1,76 +1,14 @@
using Microsoft.AspNetCore.Html;

namespace OrchardCore.Indexing;

public class DocumentIndex(string contentItemId, string contentItemVersionId)
public class DocumentIndex : DocumentIndexBase
{
public List<DocumentIndexEntry> Entries { get; } = [];

public void Set(string name, string value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, IHtmlContent value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, DateTimeOffset? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.DateTime, options));
}

public void Set(string name, int? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Integer, options));
}

public void Set(string name, bool? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Boolean, options));
}
public string ContentItemId { get; }

public void Set(string name, double? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, decimal? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, GeoPoint value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.GeoPoint, options));
}

public string ContentItemId { get; } = contentItemId;

public string ContentItemVersionId { get; } = contentItemVersionId;

public enum Types
{
Integer,
Text,
DateTime,
Boolean,
Number,
GeoPoint
}

public class GeoPoint
{
public decimal Longitude;
public decimal Latitude;
}
public string ContentItemVersionId { get; }

public class DocumentIndexEntry(string name, object value, Types type, DocumentIndexOptions options)
public DocumentIndex(string contentItemId, string contentItemVersionId)
{
public string Name { get; } = name;
public object Value { get; } = value;
public Types Type { get; } = type;
public DocumentIndexOptions Options { get; } = options;
ContentItemId = contentItemId;
ContentItemVersionId = contentItemVersionId;
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
using Microsoft.AspNetCore.Html;

namespace OrchardCore.Indexing;

public class DocumentIndexBase
{
public List<DocumentIndexEntry> Entries { get; } = [];

public void Set(string name, string value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, IHtmlContent value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Text, options));
}

public void Set(string name, DateTimeOffset? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.DateTime, options));
}

public void Set(string name, int? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Integer, options));
}

public void Set(string name, bool? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Boolean, options));
}

public void Set(string name, double? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, decimal? value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.Number, options));
}

public void Set(string name, GeoPoint value, DocumentIndexOptions options)
{
Entries.Add(new DocumentIndexEntry(name, value, Types.GeoPoint, options));
}

public enum Types
{
Integer,
Text,
DateTime,
Boolean,
Number,
GeoPoint,
}

public class GeoPoint
{
public decimal Longitude;
public decimal Latitude;
}

public class DocumentIndexEntry(string name, object value, Types type, DocumentIndexOptions options)
{
public string Name { get; } = name;
public object Value { get; } = value;
public Types Type { get; } = type;
public DocumentIndexOptions Options { get; } = options;
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
using OrchardCore.Indexing;
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Models;

Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Models;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
using OrchardCore.Indexing;
using OrchardCore.Modules;
using OrchardCore.Search.AzureAI.Models;
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Services;

Expand Down Expand Up @@ -130,7 +130,7 @@ public async Task DeleteAllDocumentsAsync(string indexName)
await DeleteDocumentsAsync(indexName, contentItemIds);
}

public async Task<bool> MergeOrUploadDocumentsAsync(string indexName, IList<DocumentIndex> indexDocuments, AzureAISearchIndexSettings indexSettings)
public async Task<bool> MergeOrUploadDocumentsAsync(string indexName, IList<DocumentIndexBase> indexDocuments, AzureAISearchIndexSettings indexSettings)
{
ArgumentException.ThrowIfNullOrEmpty(indexName);
ArgumentNullException.ThrowIfNull(indexDocuments);
Expand Down Expand Up @@ -244,21 +244,23 @@ private async Task AddIndexMappingAsync(List<AzureAISearchIndexMap> indexMapping
indexMappings.Add(indexMap);
}

private static IEnumerable<SearchDocument> CreateSearchDocuments(IEnumerable<DocumentIndex> indexDocuments, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappings)
private static IEnumerable<SearchDocument> CreateSearchDocuments(IEnumerable<DocumentIndexBase> indexDocuments, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappings)
{
foreach (var indexDocument in indexDocuments)
{
yield return CreateSearchDocument(indexDocument, mappings);
}
}

private static SearchDocument CreateSearchDocument(DocumentIndex documentIndex, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappingDictionary)
private static SearchDocument CreateSearchDocument(DocumentIndexBase documentIndex, Dictionary<string, IEnumerable<AzureAISearchIndexMap>> mappingDictionary)
{
var doc = new SearchDocument()
var doc = new SearchDocument();

if (documentIndex is DocumentIndex index)
{
{ IndexingConstants.ContentItemIdKey, documentIndex.ContentItemId },
{ IndexingConstants.ContentItemVersionIdKey, documentIndex.ContentItemVersionId },
};
doc.Add(IndexingConstants.ContentItemIdKey, index.ContentItemId);
doc.Add(IndexingConstants.ContentItemVersionIdKey, index.ContentItemVersionId);
}

foreach (var entry in documentIndex.Entries)
{
Expand All @@ -276,7 +278,7 @@ private static SearchDocument CreateSearchDocument(DocumentIndex documentIndex,

switch (entry.Type)
{
case Types.Boolean:
case DocumentIndexBase.Types.Boolean:
if (entry.Value is bool boolValue)
{
doc.TryAdd(map.AzureFieldKey, boolValue);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
using OrchardCore.Environment.Shell;
using OrchardCore.Modules;
using OrchardCore.Search.AzureAI.Models;
using static OrchardCore.Indexing.DocumentIndex;
using static OrchardCore.Indexing.DocumentIndexBase;

namespace OrchardCore.Search.AzureAI.Services;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ public async Task ProcessContentItemsAsync(params string[] indexNames)
Dictionary<string, ContentItem> allLatest = null;

// Group all DocumentIndex by index to batch update them.
var updatedDocumentsByIndex = indexSettings.ToDictionary(x => x.IndexName, b => new List<DocumentIndex>());
var updatedDocumentsByIndex = indexSettings.ToDictionary(x => x.IndexName, b => new List<DocumentIndexBase>());

var settingsByIndex = indexSettings.ToDictionary(x => x.IndexName);

Expand Down