Skip to content

Commit

Permalink
Reduce allocations from usage of enumerables
Browse files Browse the repository at this point in the history
  • Loading branch information
@kthompson
kthompson committed Jul 3, 2020
1 parent f19cbfe commit fd33173
Show file tree
Hide file tree
Showing 5 changed files with 115 additions and 62 deletions.
121 changes: 87 additions & 34 deletions src/Glob/PathTraverser.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,57 +20,110 @@ public static IEnumerable<FileSystemInfo> Traverse(this DirectoryInfo root, stri
return segments.Length == 0 ? new FileSystemInfo[0] : Traverse(root, segments, 0, cache);
}

private static readonly FileSystemInfo[] emptyFileSystemInfoArray = new FileSystemInfo[0];
private static readonly FileSystemInfo[] _emptyFileSystemInfoArray = new FileSystemInfo[0];
private static readonly DirectoryInfo[] _emptyPathJobArray = new DirectoryInfo[0];

internal static IEnumerable<FileSystemInfo> Traverse(DirectoryInfo root, Segment[] segments, int segmentIndex,
TraverseOptions options)
TraverseOptions options) =>
Traverse(new List<DirectoryInfo> { root }, segments, options);

private static IEnumerable<FileSystemInfo> Traverse(List<DirectoryInfo> roots, Segment[] segments, TraverseOptions options)
{
if (segmentIndex == segments.Length)
var segmentsLength = segments.Length;
var rootCache = new List<DirectoryInfo>();
var nextSegmentRoots = new List<DirectoryInfo>();
var segmentIndex = 0;
var emitDirectories = options.EmitDirectories;
var emitFiles = options.EmitFiles;

void Swap(ref List<DirectoryInfo> other)
{
return options.EmitDirectories
? Enumerable.Repeat<FileSystemInfo>(root, 1)
: emptyFileSystemInfoArray;
var swap = roots;
roots = other;
other = swap;
}

var segment = segments[segmentIndex];

switch (segment)
IEnumerable<DirectoryInfo> JobsMatchingSegment(DirectoryInfo directoryInfo, Segment segment)
{
case DirectorySegment directorySegment:
{
var filesToEmit =
(options.EmitFiles && segmentIndex == segments.Length - 1)
? options.GetFiles(root).Where(file => directorySegment.MatchesSegment(file.Name, options.CaseSensitive)).Cast<FileSystemInfo>()
: emptyFileSystemInfoArray;
switch (segment)
{
case DirectorySegment directorySegment:
// consume DirectorySegment
var pathJobs = (from directory in options.GetDirectories(directoryInfo)
where directorySegment.MatchesSegment(directory.Name, options.CaseSensitive)
select directory).ToArray();

nextSegmentRoots.AddRange(pathJobs);

return _emptyPathJobArray;

case DirectoryWildcard _:
{
// match zero path segments, consuming DirectoryWildcard
nextSegmentRoots.Add(directoryInfo);

// match consume 1 path segment but not the Wildcard
return options.GetDirectories(directoryInfo);
}

var dirSegmentItems = from directory in options.GetDirectories(root)
where directorySegment.MatchesSegment(directory.Name, options.CaseSensitive)
from item in Traverse(directory, segments, segmentIndex + 1, options)
select item;
default:
return _emptyPathJobArray;
}
}

return filesToEmit.Concat(dirSegmentItems);
while (true)
{
// no more segments. return all current roots
var noMoreSegments = segmentIndex == segmentsLength;
if (emitDirectories && noMoreSegments)
{
foreach (var info in roots)
{
yield return info;
}
case DirectoryWildcard _:
}

// no more roots or no more segments, go to next segment
if (roots.Count == 0 || noMoreSegments)
{
roots.Clear();
if (nextSegmentRoots.Count > 0)
{
var filesToEmit =
(options.EmitFiles && segmentIndex == segments.Length - 1)
? options.GetFiles(root).Cast<FileSystemInfo>()
: emptyFileSystemInfoArray;
Swap(ref nextSegmentRoots);
segmentIndex++;
continue;
}

// match zero path segments, consuming DirectoryWildcard
var zeroMatch = Traverse(root, segments, segmentIndex + 1, options);
yield break;
}

// match consume 1 path segment but not the Wildcard
var files = from directory in options.GetDirectories(root)
from item in Traverse(directory, segments, segmentIndex, options)
select item;
var segment = segments[segmentIndex];
var onLastSegment = segmentIndex == segmentsLength - 1;
if (emitFiles && onLastSegment)
{
var allFiles = from job in roots
let children = options.GetFiles(job)
from file in FilesMatchingSegment(children, segment, options.CaseSensitive)
select file;

return filesToEmit.Concat(zeroMatch).Concat(files);
foreach (var info in allFiles)
{
yield return info;
}
}
rootCache.Clear();
rootCache.AddRange(roots.SelectMany(job => JobsMatchingSegment(job, segment)));

default:
return emptyFileSystemInfoArray;
Swap(ref rootCache);
}
}

private static IEnumerable<FileSystemInfo> FilesMatchingSegment(IEnumerable<FileInfo> fileInfos, Segment segment, bool caseSensitive) =>
segment switch
{
DirectorySegment directorySegment => (IEnumerable<FileSystemInfo>)fileInfos.Where(file => directorySegment.MatchesSegment(file.Name, caseSensitive)),
DirectoryWildcard _ => fileInfos,
_ => _emptyFileSystemInfoArray
};
}
}
8 changes: 4 additions & 4 deletions src/Glob/TraverseOptions.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,16 +19,16 @@ public TraverseOptions(bool caseSensitive, bool emitFiles, bool emitDirectories)
public bool EmitFiles { get; }
public bool EmitDirectories { get; }

private static readonly FileInfo[] EmptyFileInfos = new FileInfo[0];
private static readonly DirectoryInfo[] EmptyDirectoryInfos = new DirectoryInfo[0];
private static readonly FileInfo[] _emptyFileInfos = new FileInfo[0];
private static readonly DirectoryInfo[] _emptyDirectoryInfos = new DirectoryInfo[0];

public virtual FileInfo[] GetFiles(DirectoryInfo root)
{
if (_fileCache.TryGetValue(root.FullName, out var cachedFiles))
return cachedFiles;

root.Refresh();
var files = root.Exists ? root.GetFiles() : EmptyFileInfos;
var files = root.Exists ? root.GetFiles() : _emptyFileInfos;
_fileCache.Add(root.FullName, files);
return files;
}
Expand All @@ -39,7 +39,7 @@ public virtual DirectoryInfo[] GetDirectories(DirectoryInfo root)
return cachedFiles;

root.Refresh();
var files = root.Exists ? root.GetDirectories() : EmptyDirectoryInfos;
var files = root.Exists ? root.GetDirectories() : _emptyDirectoryInfos;
_dirCache.Add(root.FullName, files);
return files;
}
Expand Down
16 changes: 8 additions & 8 deletions ...et.Artifacts/results/GlobExpressions.Benchmarks.GlobBenchmarks-report-github.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,11 @@ Intel Core i7-8700K CPU 3.70GHz (Coffee Lake), 1 CPU, 12 logical and 6 physical
```
| Method | Mean | Error | StdDev |
|---------------------------------------- |------------:|-----------:|-----------:|
| ParseGlob | 4,716.5 ns | 53.526 ns | 50.068 ns |
| ParseAndCompileGlob | 4,714.3 ns | 21.700 ns | 19.237 ns |
| MatchForUncompiledGlob | 5,390.9 ns | 102.690 ns | 96.056 ns |
| MatchForCompiledGlob | 591.5 ns | 2.657 ns | 2.355 ns |
| MatchForUncompiledGlobDirectoryWildcard | 5,205.7 ns | 58.186 ns | 51.580 ns |
| MatchForCompiledGlobDirectoryWildcard | 592.9 ns | 2.137 ns | 1.895 ns |
| BenchmarkParseToTree | 4,742.0 ns | 31.169 ns | 24.334 ns |
| PathTraversal | 36,948.6 ns | 661.293 ns | 618.574 ns |
| ParseGlob | 4,780.5 ns | 29.823 ns | 27.897 ns |
| ParseAndCompileGlob | 4,712.5 ns | 20.496 ns | 17.115 ns |
| MatchForUncompiledGlob | 5,386.6 ns | 42.365 ns | 39.628 ns |
| MatchForCompiledGlob | 599.5 ns | 5.919 ns | 5.537 ns |
| MatchForUncompiledGlobDirectoryWildcard | 5,245.5 ns | 42.856 ns | 40.088 ns |
| MatchForCompiledGlobDirectoryWildcard | 595.4 ns | 4.899 ns | 4.582 ns |
| BenchmarkParseToTree | 4,707.7 ns | 50.401 ns | 47.145 ns |
| PathTraversal | 38,026.5 ns | 361.679 ns | 320.619 ns |
16 changes: 8 additions & 8 deletions ...ks/BenchmarkDotNet.Artifacts/results/GlobExpressions.Benchmarks.GlobBenchmarks-report.csv
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
Method,Job,AnalyzeLaunchVariance,EvaluateOverhead,MaxAbsoluteError,MaxRelativeError,MinInvokeCount,MinIterationTime,OutlierMode,Affinity,EnvironmentVariables,Jit,Platform,Runtime,AllowVeryLargeObjects,Concurrent,CpuGroups,Force,HeapAffinitizeMask,HeapCount,NoAffinitize,RetainVm,Server,Arguments,BuildConfiguration,Clock,EngineFactory,Toolchain,IsMutator,InvocationCount,IterationCount,IterationTime,LaunchCount,MaxIterationCount,MaxWarmupIterationCount,MinIterationCount,MinWarmupIterationCount,RunStrategy,UnrollFactor,WarmupCount,Mean,Error,StdDev
ParseGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"4,716.5 ns",53.526 ns,50.068 ns
ParseAndCompileGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"4,714.3 ns",21.700 ns,19.237 ns
MatchForUncompiledGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"5,390.9 ns",102.690 ns,96.056 ns
MatchForCompiledGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,591.5 ns,2.657 ns,2.355 ns
MatchForUncompiledGlobDirectoryWildcard,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"5,205.7 ns",58.186 ns,51.580 ns
MatchForCompiledGlobDirectoryWildcard,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,592.9 ns,2.137 ns,1.895 ns
BenchmarkParseToTree,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"4,742.0 ns",31.169 ns,24.334 ns
PathTraversal,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"36,948.6 ns",661.293 ns,618.574 ns
ParseGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"4,780.5 ns",29.823 ns,27.897 ns
ParseAndCompileGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"4,712.5 ns",20.496 ns,17.115 ns
MatchForUncompiledGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"5,386.6 ns",42.365 ns,39.628 ns
MatchForCompiledGlob,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,599.5 ns,5.919 ns,5.537 ns
MatchForUncompiledGlobDirectoryWildcard,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"5,245.5 ns",42.856 ns,40.088 ns
MatchForCompiledGlobDirectoryWildcard,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,595.4 ns,4.899 ns,4.582 ns
BenchmarkParseToTree,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"4,707.7 ns",50.401 ns,47.145 ns
PathTraversal,Default,False,Default,Default,Default,Default,Default,Default,111111111111,Empty,RyuJit,X64,Core,False,True,False,True,Default,Default,False,False,False,Default,Default,Default,Default,Default,Default,1,Default,Default,Default,Default,Default,Default,Default,Default,16,Default,"38,026.5 ns",361.679 ns,320.619 ns
16 changes: 8 additions & 8 deletions ...s/BenchmarkDotNet.Artifacts/results/GlobExpressions.Benchmarks.GlobBenchmarks-report.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,14 +24,14 @@
<table>
<thead><tr><th> Method</th><th> Mean</th><th>Error</th><th>StdDev</th>
</tr>
</thead><tbody><tr><td>ParseGlob</td><td>4,716.5 ns</td><td>53.526 ns</td><td>50.068 ns</td>
</tr><tr><td>ParseAndCompileGlob</td><td>4,714.3 ns</td><td>21.700 ns</td><td>19.237 ns</td>
</tr><tr><td>MatchForUncompiledGlob</td><td>5,390.9 ns</td><td>102.690 ns</td><td>96.056 ns</td>
</tr><tr><td>MatchForCompiledGlob</td><td>591.5 ns</td><td>2.657 ns</td><td>2.355 ns</td>
</tr><tr><td>MatchForUncompiledGlobDirectoryWildcard</td><td>5,205.7 ns</td><td>58.186 ns</td><td>51.580 ns</td>
</tr><tr><td>MatchForCompiledGlobDirectoryWildcard</td><td>592.9 ns</td><td>2.137 ns</td><td>1.895 ns</td>
</tr><tr><td>BenchmarkParseToTree</td><td>4,742.0 ns</td><td>31.169 ns</td><td>24.334 ns</td>
</tr><tr><td>PathTraversal</td><td>36,948.6 ns</td><td>661.293 ns</td><td>618.574 ns</td>
</thead><tbody><tr><td>ParseGlob</td><td>4,780.5 ns</td><td>29.823 ns</td><td>27.897 ns</td>
</tr><tr><td>ParseAndCompileGlob</td><td>4,712.5 ns</td><td>20.496 ns</td><td>17.115 ns</td>
</tr><tr><td>MatchForUncompiledGlob</td><td>5,386.6 ns</td><td>42.365 ns</td><td>39.628 ns</td>
</tr><tr><td>MatchForCompiledGlob</td><td>599.5 ns</td><td>5.919 ns</td><td>5.537 ns</td>
</tr><tr><td>MatchForUncompiledGlobDirectoryWildcard</td><td>5,245.5 ns</td><td>42.856 ns</td><td>40.088 ns</td>
</tr><tr><td>MatchForCompiledGlobDirectoryWildcard</td><td>595.4 ns</td><td>4.899 ns</td><td>4.582 ns</td>
</tr><tr><td>BenchmarkParseToTree</td><td>4,707.7 ns</td><td>50.401 ns</td><td>47.145 ns</td>
</tr><tr><td>PathTraversal</td><td>38,026.5 ns</td><td>361.679 ns</td><td>320.619 ns</td>
</tr></tbody></table>
</body>
</html>

0 comments on commit fd33173

Please sign in to comment.