diff --git a/.github/workflows/CodeQL Security Scan.yml b/.github/workflows/CodeQL Security Scan.yml index 0762ff435..b19d39a23 100644 --- a/.github/workflows/CodeQL Security Scan.yml +++ b/.github/workflows/CodeQL Security Scan.yml @@ -79,7 +79,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Lib edFi.admin.dataaccess.yml b/.github/workflows/Lib edFi.admin.dataaccess.yml index be280ac1d..51ecec682 100644 --- a/.github/workflows/Lib edFi.admin.dataaccess.yml +++ b/.github/workflows/Lib edFi.admin.dataaccess.yml @@ -45,7 +45,7 @@ jobs: working-directory: ./.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Lib edFi.common.yml b/.github/workflows/Lib edFi.common.yml index a9ec8717f..d7804ba24 100644 --- a/.github/workflows/Lib edFi.common.yml +++ b/.github/workflows/Lib edFi.common.yml @@ -45,7 +45,7 @@ jobs: working-directory: ./.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Lib edFi.loadtools.yml b/.github/workflows/Lib edFi.loadtools.yml index 42242fb0e..f6ecb095a 100644 --- a/.github/workflows/Lib edFi.loadtools.yml +++ b/.github/workflows/Lib edFi.loadtools.yml @@ -45,7 +45,7 @@ jobs: working-directory: ./.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Lib edFi.ods.api.yml b/.github/workflows/Lib edFi.ods.api.yml index 8e248f56b..bf4d1c0d1 100644 --- a/.github/workflows/Lib edFi.ods.api.yml +++ b/.github/workflows/Lib edFi.ods.api.yml @@ -45,7 +45,7 @@ jobs: working-directory: ./.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Lib edFi.ods.common.yml b/.github/workflows/Lib edFi.ods.common.yml index b006ab6c2..600fa62e2 100644 --- a/.github/workflows/Lib edFi.ods.common.yml +++ b/.github/workflows/Lib edFi.ods.common.yml @@ -45,7 +45,7 @@ jobs: working-directory: ./.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Lib edFi.ods.standard.yml b/.github/workflows/Lib edFi.ods.standard.yml index f84ba6726..6750e3a53 100644 --- a/.github/workflows/Lib edFi.ods.standard.yml +++ b/.github/workflows/Lib edFi.ods.standard.yml @@ -67,7 +67,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Lib edFi.security.dataaccess.yml b/.github/workflows/Lib edFi.security.dataaccess.yml index 54c4c8c6c..4c916cbc0 100644 --- a/.github/workflows/Lib edFi.security.dataaccess.yml +++ b/.github/workflows/Lib edFi.security.dataaccess.yml @@ -45,7 +45,7 @@ jobs: working-directory: ./.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.CodeGen.yml b/.github/workflows/Pkg EdFi.Ods.CodeGen.yml index e9bff9ec0..fbd5721a3 100644 --- a/.github/workflows/Pkg EdFi.Ods.CodeGen.yml +++ b/.github/workflows/Pkg EdFi.Ods.CodeGen.yml @@ -73,7 +73,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.PostgreSQL.yml b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.PostgreSQL.yml index 7937fab7e..5dadabe9d 100644 --- a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.PostgreSQL.yml +++ b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.PostgreSQL.yml @@ -110,7 +110,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.PostgreSQL.yml b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.PostgreSQL.yml index ffc20af91..d4dc46433 100644 --- a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.PostgreSQL.yml +++ b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.PostgreSQL.yml @@ -130,7 +130,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.yml b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.yml index 3fcd3e62b..80a75b847 100644 --- a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.yml +++ b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.TPDM.yml @@ -129,7 +129,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.yml b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.yml index 392b39e9f..4eebee305 100644 --- a/.github/workflows/Pkg EdFi.Ods.Minimal.Template.yml +++ b/.github/workflows/Pkg EdFi.Ods.Minimal.Template.yml @@ -115,7 +115,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Populated.Template.PostgreSQL.yml b/.github/workflows/Pkg EdFi.Ods.Populated.Template.PostgreSQL.yml index 75392908d..46efe0bdd 100644 --- a/.github/workflows/Pkg EdFi.Ods.Populated.Template.PostgreSQL.yml +++ b/.github/workflows/Pkg EdFi.Ods.Populated.Template.PostgreSQL.yml @@ -109,7 +109,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.PostgreSQL.yml b/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.PostgreSQL.yml index a4d88f598..e18fcbace 100644 --- a/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.PostgreSQL.yml +++ b/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.PostgreSQL.yml @@ -130,7 +130,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.yml b/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.yml index 26f6425f0..15f172005 100644 --- a/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.yml +++ b/.github/workflows/Pkg EdFi.Ods.Populated.Template.TPDM.yml @@ -129,7 +129,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.Ods.Populated.Template.yml b/.github/workflows/Pkg EdFi.Ods.Populated.Template.yml index 25ed16df1..fe0999536 100644 --- a/.github/workflows/Pkg EdFi.Ods.Populated.Template.yml +++ b/.github/workflows/Pkg EdFi.Ods.Populated.Template.yml @@ -107,7 +107,7 @@ jobs: working-directory: ./Ed-Fi-ODS/.github/workflows run: sudo ./install-mono.sh - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/.github/workflows/Pkg EdFi.ProjectTemplates.Installer.yml b/.github/workflows/Pkg EdFi.ProjectTemplates.Installer.yml index 26dc524c0..9b7c00b42 100644 --- a/.github/workflows/Pkg EdFi.ProjectTemplates.Installer.yml +++ b/.github/workflows/Pkg EdFi.ProjectTemplates.Installer.yml @@ -55,7 +55,7 @@ jobs: - name: Setup MSBuild for .NET Framework 4.8 uses: microsoft/setup-msbuild@6fb02220983dee41ce7ae257b6f4d8f9bf5ed4ce #v2.0.0 - name: Cache Nuget packages - uses: actions/cache@ab5e6d0c87105b4c9c2047343972218f562e4319 #v4.0.1 + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 #v4.2 with: path: ~/.nuget/packages key: ${{ runner.os }}-nuget-${{ hashFiles('**/*.csproj*', '**/configuration.packages.json') }} diff --git a/Application/EdFi.Ods.Common/Repositories/EntityDeserializer.cs b/Application/EdFi.Ods.Common/Repositories/EntityDeserializer.cs index 96f1368de..ac8378ef9 100644 --- a/Application/EdFi.Ods.Common/Repositories/EntityDeserializer.cs +++ b/Application/EdFi.Ods.Common/Repositories/EntityDeserializer.cs @@ -3,6 +3,8 @@ // The Ed-Fi Alliance licenses this file to you under the Apache License, Version 2.0. // See the LICENSE and NOTICES files in the project root for more information. +using System; +using System.Threading; using System.Threading.Tasks; using EdFi.Ods.Common.Context; using EdFi.Ods.Common.Infrastructure; @@ -24,6 +26,10 @@ public class EntityDeserializer : IEntityDeserializer private readonly ILog _logger = LogManager.GetLogger(typeof(EntityDeserializer)); + private long _deserializationFailedCount; + private DateTime _lastDeserializationWarningTime; + private readonly TimeSpan _deserializationWarningPeriod = TimeSpan.FromMinutes(5); + public EntityDeserializer( ISurrogateIdMutator[] surrogateIdMutators, ISessionFactory sessionFactory, @@ -48,6 +54,30 @@ public async Task DeserializeAsync(IItemRawData itemRawData) // Deserialize the entity entity = MessagePackHelper.DecompressAndDeserializeAggregate(itemRawData.AggregateData); } + catch (Exception ex) + { + Interlocked.Increment(ref _deserializationFailedCount); + + // Determine if "quiet" period for logging warnings has elapsed + if (DateTime.Now > (_lastDeserializationWarningTime + _deserializationWarningPeriod)) + { + // Reset the tracking variables + _lastDeserializationWarningTime = DateTime.Now; + var failedCount = Interlocked.Exchange(ref _deserializationFailedCount, 0); + + string messageCountAddendum = null; + + if (failedCount > 1) + { + messageCountAddendum = $" and {failedCount - 1:N0} other item(s) since last warning"; + } + + // Prevent exceptions during deserialization from failing the processing -- revert to returning a null instance + _logger.Warn($"Unable to deserialize entity of type '{typeof(TEntity).Name}' (with AggregateId of {itemRawData.AggregateId}){messageCountAddendum}. Falling back to load through NHibernate repository. HINT: Serialized data is in an inconsistent state. Clear the values in the 'AggregateData' column for the affected records. Next warning won't occur until '{_lastDeserializationWarningTime.Add(_deserializationWarningPeriod)}'...", ex); + } + + return default; + } finally { // Indicate that deserialization is finished