Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Mitigations for dll load problem #57

Merged
merged 2 commits into from
Jun 22, 2021
Merged

Mitigations for dll load problem #57

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2c57662
remove obsolete code for query support.
sebastianburckhardt Jun 22, 2021
e2ddd4a
add fast-fail for dll loading problem (#55), and go back to last func…
sebastianburckhardt Jun 22, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion samples/Hello/Hello.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Azure.DurableTask.Netherite.AzureFunctions" Version="0.3.0-alpha" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.12" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.3" />
</ItemGroup>
<ItemGroup>
<None Update="host.json">
Expand Down
6 changes: 6 additions & 0 deletions src/DurableTask.Netherite/OrchestrationService/NetheriteOrchestrationService.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,12 @@ public NetheriteOrchestrationService(NetheriteOrchestrationServiceSettings setti
EtwSource.Log.OrchestrationServiceCreated(this.ServiceInstanceId, this.StorageAccountName, this.Settings.HubName, this.Settings.WorkerId, TraceUtils.AppName, TraceUtils.ExtensionVersion);
this.Logger.LogInformation("NetheriteOrchestrationService created, workerId={workerId}, processorCount={processorCount}, transport={transport}, storage={storage}", this.Settings.WorkerId, Environment.ProcessorCount, this.configuredTransport, this.configuredStorage);

if (this.configuredStorage == TransportConnectionString.StorageChoices.Faster)
{
// force dll load here so exceptions are observed early
var _ = System.Threading.Channels.Channel.CreateBounded<DateTime>(10);
}

switch (this.configuredTransport)
{
case TransportConnectionString.TransportChoices.Memory:
Expand Down
5 changes: 0 additions & 5 deletions src/DurableTask.Netherite/OrchestrationService/NetheriteOrchestrationServiceSettings.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -139,11 +139,6 @@ public class NetheriteOrchestrationServiceSettings
/// </summary>
public long MaxTimeMsBetweenCheckpoints { get; set; } = 60 * 1000;

/// <summary>
/// Whether to use the Faster PSF support for handling queries.
/// </summary>
public bool UsePSFQueries { get; set; } = false;

/// <summary>
/// Set this to a local file path to make FASTER use local files instead of blobs. Currently,
/// this makes sense only for local testing and debugging.
Expand Down
37 changes: 1 addition & 36 deletions src/DurableTask.Netherite/StorageProviders/Faster/AzureBlobs/BlobManager.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,6 @@ public static string GetStorageFormat(NetheriteOrchestrationServiceSettings sett
return JsonConvert.SerializeObject(new StorageFormatSettings()
{
UseAlternateObjectStore = settings.UseAlternateObjectStore,
UsePSFQueries = settings.UsePSFQueries,
FormatVersion = StorageFormatVersion,
},
serializerSettings);
Expand All @@ -120,16 +119,14 @@ public static string GetStorageFormat(NetheriteOrchestrationServiceSettings sett
class StorageFormatSettings
{
// this must stay the same

[JsonProperty("FormatVersion")]
public int FormatVersion { get; set; }

// the following can be changed between versions

[JsonProperty("UseAlternateObjectStore", DefaultValueHandling=DefaultValueHandling.Ignore)]
public bool? UseAlternateObjectStore { get; set; }

[JsonProperty("UsePSFQueries", DefaultValueHandling = DefaultValueHandling.Ignore)]
public bool? UsePSFQueries { get; set; }
}

static readonly JsonSerializerSettings serializerSettings = new JsonSerializerSettings()
Expand All @@ -140,7 +137,6 @@ class StorageFormatSettings
Formatting = Formatting.None,
};


public static void CheckStorageFormat(string format, NetheriteOrchestrationServiceSettings settings)
{
try
Expand All @@ -151,10 +147,6 @@ public static void CheckStorageFormat(string format, NetheriteOrchestrationServi
{
throw new InvalidOperationException("The Netherite configuration setting 'UseAlternateObjectStore' is incompatible with the existing taskhub.");
}
if (taskhubFormat.UsePSFQueries != settings.UsePSFQueries)
{
throw new InvalidOperationException("The Netherite configuration setting 'UsePSFQueries' is incompatible with the existing taskhub.");
}
if (taskhubFormat.FormatVersion != StorageFormatVersion)
{
throw new InvalidOperationException($"The current storage format version (={StorageFormatVersion}) is incompatible with the existing taskhub (={taskhubFormat.FormatVersion}).");
Expand Down Expand Up @@ -182,33 +174,6 @@ public void PurgeAll()
CheckPointType = CheckpointType.FoldOver
};

#if FASTER_SUPPORTS_PSF
internal PSFRegistrationSettings<TKey> CreatePSFRegistrationSettings<TKey>(uint numberPartitions, int groupOrdinal)
{
var storeLogSettings = this.StoreLogSettings(false, numberPartitions);
return new PSFRegistrationSettings<TKey>
{
HashTableSize = FasterKV.HashTableSize,
LogSettings = new LogSettings()
{
LogDevice = this.PsfLogDevices[groupOrdinal],
PageSizeBits = storeLogSettings.PageSizeBits,
SegmentSizeBits = storeLogSettings.SegmentSizeBits,
MemorySizeBits = storeLogSettings.MemorySizeBits,
CopyReadsToTail = false,
ReadCacheSettings = storeLogSettings.ReadCacheSettings
},
CheckpointSettings = new CheckpointSettings
{
CheckpointManager = this.UseLocalFilesForTestingAndDebugging
? new LocalFileCheckpointManager(this.PsfCheckpointInfos[groupOrdinal], this.LocalPsfCheckpointDirectoryPath(groupOrdinal), this.GetCheckpointCompletedBlobName())
: (ICheckpointManager)new PsfBlobCheckpointManager(this, groupOrdinal),
CheckPointType = CheckpointType.FoldOver
}
};
}
#endif

public const int MaxRetries = 10;

public static BlobRequestOptions BlobRequestOptionsAggressiveTimeout = new BlobRequestOptions()
Expand Down
85 changes: 1 addition & 84 deletions src/DurableTask.Netherite/StorageProviders/Faster/FasterKV.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,6 @@ class FasterKV : TrackedObjectStore

ClientSession<Key, Value, EffectTracker, TrackedObject, object, IFunctions<Key, Value, EffectTracker, TrackedObject, object>> mainSession;

#if FASTER_SUPPORTS_PSF
// We currently place all PSFs into a single group with a single TPSFKey type
internal const int PSFCount = 1;

internal IPSF RuntimeStatusPsf;
internal IPSF CreatedTimePsf;
internal IPSF InstanceIdPrefixPsf;
#endif
public FasterKV(Partition partition, BlobManager blobManager)
{
this.partition = partition;
Expand All @@ -49,26 +41,6 @@ public FasterKV(Partition partition, BlobManager blobManager)
valueSerializer = () => new Value.Serializer(this.StoreStats),
});

#if FASTER_SUPPORTS_PSF
if (partition.Settings.UsePSFQueries)
{
int groupOrdinal = 0;
var psfs = fht.RegisterPSF(this.blobManager.CreatePSFRegistrationSettings<PSFKey>(partition.NumberPartitions(), groupOrdinal++),
(nameof(this.RuntimeStatusPsf), (k, v) => v.Val is InstanceState state
? (PSFKey?)new PSFKey(state.OrchestrationState.OrchestrationStatus)
: null),
(nameof(this.CreatedTimePsf), (k, v) => v.Val is InstanceState state
? (PSFKey?)new PSFKey(state.OrchestrationState.CreatedTime)
: null),
(nameof(this.InstanceIdPrefixPsf), (k, v) => v.Val is InstanceState state
? (PSFKey?)new PSFKey(state.InstanceId)
: null));

this.RuntimeStatusPsf = psfs[0];
this.CreatedTimePsf = psfs[1];
this.InstanceIdPrefixPsf = psfs[2];
}
#endif
this.terminationToken = partition.ErrorHandler.Token;

var _ = this.terminationToken.Register(
Expand Down Expand Up @@ -232,66 +204,11 @@ public override async Task QueryAsync(PartitionQueryEvent queryEvent, EffectTrac
{
var instanceQuery = queryEvent.InstanceQuery;

#if FASTER_SUPPORTS_PSF
IAsyncEnumerable<OrchestrationState> queryPSFsAsync(ClientSession<Key, Value, EffectTracker, TrackedObject, PartitionReadEvent, Functions> session)
{
// Issue the PSF query. Note that pending operations will be completed before this returns.
var querySpec = new List<(IPSF, IEnumerable<PSFKey>)>();
if (instanceQuery.HasRuntimeStatus)
querySpec.Add((this.RuntimeStatusPsf, instanceQuery.RuntimeStatus.Select(s => new PSFKey(s))));
if (instanceQuery.CreatedTimeFrom.HasValue || instanceQuery.CreatedTimeTo.HasValue)
{
IEnumerable<PSFKey> enumerateDateBinKeys()
{
var to = instanceQuery.CreatedTimeTo ?? DateTime.UtcNow;
var from = instanceQuery.CreatedTimeFrom ?? to.AddDays(-7); // TODO Some default so we don't have to iterate from the first possible date
for (var dt = from; dt <= to; dt += PSFKey.DateBinInterval)
yield return new PSFKey(dt);
}
querySpec.Add((this.CreatedTimePsf, enumerateDateBinKeys()));
}
if (!string.IsNullOrWhiteSpace(instanceQuery.InstanceIdPrefix))
querySpec.Add((this.InstanceIdPrefixPsf, new[] { new PSFKey(instanceQuery.InstanceIdPrefix) }));
var querySettings = new PSFQuerySettings
{
// This is a match-all-PSFs enumeration so do not continue after any PSF has hit EOS
OnStreamEnded = (unusedPsf, unusedIndex) => false
};

OrchestrationState getOrchestrationState(ref Value v)
{
if (v.Val is byte[] serialized)
{
var result = ((InstanceState)Serializer.DeserializeTrackedObject(serialized))?.OrchestrationState;
if (result != null && !instanceQuery.FetchInput)
{
result.Input = null;
}
return result;
}
else
{
var state = ((InstanceState)((TrackedObject)v))?.OrchestrationState;
var result = state?.ClearFieldsImmutably(instanceQuery.FetchInput, true);
return result;
}
}

return session.QueryPSFAsync(querySpec, matches => matches.All(b => b), querySettings)
.Select(providerData => getOrchestrationState(ref providerData.GetValue()))
.Where(orchestrationState => orchestrationState != null);
}
#else
IAsyncEnumerable<OrchestrationState> queryPSFsAsync(ClientSession<Key, Value, EffectTracker, TrackedObject, object, IFunctions<Key, Value, EffectTracker, TrackedObject, object>> session)
=> this.ScanOrchestrationStates(effectTracker, queryEvent);
#endif
// create an individual session for this query so the main session can be used
// while the query is progressing.
using (var session = this.CreateASession())
{
var orchestrationStates = (this.partition.Settings.UsePSFQueries && instanceQuery.IsSet)
? queryPSFsAsync(session)
: this.ScanOrchestrationStates(effectTracker, queryEvent);
var orchestrationStates = this.ScanOrchestrationStates(effectTracker, queryEvent);

await effectTracker.ProcessQueryResultAsync(queryEvent, orchestrationStates);
}
Expand Down
4 changes: 0 additions & 4 deletions src/DurableTask.Netherite/StorageProviders/Faster/FasterStorage.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,11 +64,7 @@ public async Task<long> CreateOrRestoreAsync(Partition partition, IPartitionErro
this.partition = partition;
this.terminationToken = errorHandler.Token;

#if FASTER_SUPPORTS_PSF
int psfCount = partition.Settings.UsePSFQueries ? FasterKV.PSFCount : 0;
#else
int psfCount = 0;
#endif

this.blobManager = new BlobManager(
this.storageAccount,
Expand Down
2 changes: 1 addition & 1 deletion test/PerformanceTests/PerformanceTests.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.DurableTask" Version="2.5.0" />
<PackageReference Include="Microsoft.Azure.DurableTask.Core" Version="2.5.5" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.EventHubs" Version="4.2.0" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.13" />
<PackageReference Include="Microsoft.NET.Sdk.Functions" Version="3.0.3" />
<PackageReference Include="Microsoft.Azure.WebJobs.Extensions.Storage" Version="4.0.4" />
</ItemGroup>
<ItemGroup>
Expand Down