Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove secondary caching in ContentDefinitionManager, improve error message in DataMigrationManager #16212

Closed
wants to merge 30 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
30 commits
Select commit Hold shift + click to select a range
7fda7cb
try fix ConcurrencyException
hyzx86 Jun 2, 2024
18386eb
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 2, 2024
0316bd9
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 4, 2024
c95057b
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 5, 2024
5050392
fix dependencies
hyzx86 Jun 5, 2024
a2f42c3
fix startup
hyzx86 Jun 5, 2024
fd79bad
clear
hyzx86 Jun 5, 2024
b4bd1a8
fix RequireFeatures
hyzx86 Jun 5, 2024
b26f81e
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 5, 2024
fa4483a
restore Feature on MigrationFile
hyzx86 Jun 5, 2024
36fbf40
Merge branch 'TypeUpdateConcurrencyException' of https://github.com/h…
hyzx86 Jun 5, 2024
414f1ef
update
hyzx86 Jun 5, 2024
55541bc
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 5, 2024
4fee32f
update MigrationName
hyzx86 Jun 5, 2024
4345ab3
Merge branch 'TypeUpdateConcurrencyException' of https://github.com/h…
hyzx86 Jun 5, 2024
0de24d1
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 5, 2024
b3ef613
fix migrationManger
hyzx86 Jun 5, 2024
739ecc1
Merge branch 'TypeUpdateConcurrencyException' of https://github.com/h…
hyzx86 Jun 5, 2024
f4282c7
clear
hyzx86 Jun 5, 2024
d83450e
restore changes
hyzx86 Jun 5, 2024
f0e3ba6
clear code
hyzx86 Jun 5, 2024
173f2d8
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 6, 2024
4185e4e
recurrence of problems
hyzx86 Jun 6, 2024
d1736cc
Remove L2 cache
hyzx86 Jun 6, 2024
68fe0f4
update use latest filter
hyzx86 Jun 6, 2024
bf3945f
restore changes
hyzx86 Jun 6, 2024
3514bb9
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 6, 2024
70c09c8
restore changes
hyzx86 Jun 6, 2024
204077e
Merge branch 'TypeUpdateConcurrencyException' of https://github.com/h…
hyzx86 Jun 6, 2024
7033644
Merge branch 'main' into TypeUpdateConcurrencyException
hyzx86 Jun 6, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using System;
using Microsoft.Extensions.Caching.Memory;
using OrchardCore.ContentManagement.Metadata;
using OrchardCore.ContentManagement.Metadata.Models;
Expand All @@ -19,21 +18,13 @@ public class ContentDefinitionManager : IContentDefinitionManager
private readonly IContentDefinitionStore _contentDefinitionStore;
private readonly IMemoryCache _memoryCache;

private readonly ConcurrentDictionary<string, ContentTypeDefinition> _cachedTypeDefinitions;
private readonly ConcurrentDictionary<string, ContentPartDefinition> _cachedPartDefinitions;

private readonly Dictionary<string, ContentTypeDefinition> _scopedTypeDefinitions = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, ContentPartDefinition> _scopedPartDefinitions = new(StringComparer.OrdinalIgnoreCase);

public ContentDefinitionManager(
IContentDefinitionStore contentDefinitionStore,
IMemoryCache memoryCache)
{
_contentDefinitionStore = contentDefinitionStore;
_memoryCache = memoryCache;

_cachedTypeDefinitions = _memoryCache.GetOrCreate("TypeDefinitions", entry => new ConcurrentDictionary<string, ContentTypeDefinition>(StringComparer.OrdinalIgnoreCase));
_cachedPartDefinitions = _memoryCache.GetOrCreate("PartDefinitions", entry => new ConcurrentDictionary<string, ContentPartDefinition>(StringComparer.OrdinalIgnoreCase));
}

public async Task<IEnumerable<ContentTypeDefinition>> LoadTypeDefinitionsAsync()
Expand Down Expand Up @@ -72,11 +63,6 @@ public async Task<ContentTypeDefinition> LoadTypeDefinitionAsync(string name)
{
ArgumentException.ThrowIfNullOrEmpty(name);

if (_scopedTypeDefinitions.TryGetValue(name, out var typeDefinition))
{
return typeDefinition;
}
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@sebastienros should be the problem here, the Loadxx family of methods should no longer fetch content from the cache
/cc @MikeAlhayek

Copy link
Contributor Author

@hyzx86 hyzx86 Jun 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hi @MikeAlhayek , do I need to revert these changes? Or do I keep them?
Because there is already a cache in the ContentDefinitionStore and a second-level cache would make the logic more complex

Copy link
Contributor Author

@hyzx86 hyzx86 Jun 6, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we use both MemoryCache and Redis caching, it may have problems enabling distributed caching and multi-server deployments


var document = await _contentDefinitionStore.LoadContentDefinitionAsync();

return LoadTypeDefinition(document, name);
Expand All @@ -97,11 +83,6 @@ public async Task<ContentPartDefinition> LoadPartDefinitionAsync(string name)
{
ArgumentException.ThrowIfNullOrEmpty(name);

if (_scopedPartDefinitions.TryGetValue(name, out var partDefinition))
{
return partDefinition;
}

var document = await _contentDefinitionStore.LoadContentDefinitionAsync();

return LoadPartDefinition(document, name);
Expand Down Expand Up @@ -182,27 +163,20 @@ public async Task StorePartDefinitionAsync(ContentPartDefinition contentPartDefi

public async Task<string> GetIdentifierAsync() => (await _contentDefinitionStore.GetContentDefinitionAsync()).Identifier;

private ContentTypeDefinition LoadTypeDefinition(ContentDefinitionRecord document, string name) =>
!_scopedTypeDefinitions.TryGetValue(name, out var typeDefinition)
? _scopedTypeDefinitions[name] = Build(
private static ContentTypeDefinition LoadTypeDefinition(ContentDefinitionRecord document, string name) => Build(
document.ContentTypeDefinitionRecords.FirstOrDefault(type => type.Name.EqualsOrdinalIgnoreCase(name)),
document.ContentPartDefinitionRecords)
: typeDefinition;
document.ContentPartDefinitionRecords);

private ContentTypeDefinition GetTypeDefinition(ContentDefinitionRecord document, string name) =>
_cachedTypeDefinitions.GetOrAdd(name, name => Build(
private static ContentTypeDefinition GetTypeDefinition(ContentDefinitionRecord document, string name) =>
Build(
document.ContentTypeDefinitionRecords.FirstOrDefault(type => type.Name.EqualsOrdinalIgnoreCase(name)),
document.ContentPartDefinitionRecords));
document.ContentPartDefinitionRecords);

private ContentPartDefinition LoadPartDefinition(ContentDefinitionRecord document, string name) =>
!_scopedPartDefinitions.TryGetValue(name, out var partDefinition)
? _scopedPartDefinitions[name] = Build(
document.ContentPartDefinitionRecords.FirstOrDefault(part => part.Name.EqualsOrdinalIgnoreCase(name)))
: partDefinition;
private static ContentPartDefinition LoadPartDefinition(ContentDefinitionRecord document, string name) =>
Build(document.ContentPartDefinitionRecords.FirstOrDefault(part => part.Name.EqualsOrdinalIgnoreCase(name)));

private ContentPartDefinition GetPartDefinition(ContentDefinitionRecord document, string name) =>
_cachedPartDefinitions.GetOrAdd(name, name => Build(
document.ContentPartDefinitionRecords.FirstOrDefault(record => record.Name.EqualsOrdinalIgnoreCase(name))));
private static ContentPartDefinition GetPartDefinition(ContentDefinitionRecord document, string name) => Build(
document.ContentPartDefinitionRecords.FirstOrDefault(record => record.Name.EqualsOrdinalIgnoreCase(name)));

private static ContentTypeDefinitionRecord Acquire(
ContentDefinitionRecord document,
Expand Down Expand Up @@ -376,10 +350,6 @@ private static ContentFieldDefinition Build(ContentFieldDefinitionRecord source)
private async Task UpdateContentDefinitionRecordAsync(ContentDefinitionRecord document)
{
await _contentDefinitionStore.SaveContentDefinitionAsync(document);

// If multiple updates in the same scope, types and parts may need to be rebuilt.
_scopedTypeDefinitions.Clear();
_scopedPartDefinitions.Clear();
}

/// <summary>
Expand All @@ -393,10 +363,6 @@ private void CheckDocumentIdentifier(ContentDefinitionRecord document)
{
Identifier = document.Identifier,
};

_cachedTypeDefinitions.Clear();
_cachedPartDefinitions.Clear();

_memoryCache.Set(CacheKey, cacheEntry);
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,8 @@ private async Task UpdateAsync(string featureId)
// Get current version for this migration.
var dataMigrationRecord = await GetDataMigrationRecordAsync(tempMigration);

var migrationName = migration.GetType().FullName;

var current = 0;
if (dataMigrationRecord != null)
{
Expand All @@ -195,7 +197,7 @@ private async Task UpdateAsync(string featureId)
}
else
{
dataMigrationRecord = new Records.DataMigration { DataMigrationClass = migration.GetType().FullName };
dataMigrationRecord = new Records.DataMigration { DataMigrationClass = migrationName };
_dataMigrationRecord.DataMigrations.Add(dataMigrationRecord);
}

Expand All @@ -209,7 +211,7 @@ private async Task UpdateAsync(string featureId)

if (createMethod == null)
{
_logger.LogWarning("The migration '{name}' for '{FeatureName}' does not contain a proper Create or CreateAsync method.", migration.GetType().FullName, featureId);
_logger.LogWarning("The migration '{MigrationName}' for '{FeatureName}' does not contain a proper Create or CreateAsync method.", migrationName, featureId);
continue;
}

Expand All @@ -220,7 +222,7 @@ private async Task UpdateAsync(string featureId)

while (lookupTable.TryGetValue(current, out var methodInfo))
{
_logger.LogInformation("Applying migration for '{FeatureName}' from version {Version}.", featureId, current);
_logger.LogInformation("Applying migration '{MigrationName}' for '{FeatureName}' from version {Version}.", migrationName, featureId, current);

current = await InvokeMethodAsync(methodInfo, migration);
}
Expand All @@ -235,7 +237,7 @@ private async Task UpdateAsync(string featureId)
}
catch (Exception ex)
{
_logger.LogError(ex, "Error while running migration version {Version} for '{FeatureName}'.", current, featureId);
_logger.LogError(ex, "Error while running migration '{MigrationName}' version {Version} for '{FeatureName}'.", migrationName, current, featureId);

await _session.CancelAsync();
}
Expand Down