Skip to content

Commit

Permalink
Parallelization Stability Improvements (#216)
Browse files Browse the repository at this point in the history 
* Fix more parallel edge cases

* Only read through tags list once.

* threadsafe metrictagcounter.count

* Revert counter only tag change

* Update all related counters

* Clean up

* Partial revert change to AnalyzeHtmlWriter

ConcurrentStack was overkill here.

* Remove unused dependency.
  • Loading branch information
@gfs
gfs authored May 21, 2020
1 parent e976a5c commit fdc1f87
Show file tree
Hide file tree
Showing 5 changed files with 59 additions and 69 deletions.
14 changes: 6 additions & 8 deletions AppInspector.CLI/Writers/AnalyzeHtmlWriter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,19 +507,17 @@ private List<TagInfo> GetTagInfoListBySeverity()
/// <summary>
/// Opportunity for any final data prep before report gen
/// </summary>
public List<TagCounterUI> ConvertTagCounters(List<MetricTagCounter> metricTagCounters)
public List<TagCounterUI> ConvertTagCounters(IEnumerable<MetricTagCounter> metricTagCounters)
{
List<TagCounterUI> result = new List<TagCounterUI>();
//TagCountersUI is liquid compatible while TagCounters is not to support json serialization; the split prevents exception
//not fixable via json iteration disabling
foreach (MetricTagCounter counter in metricTagCounters)

result.AddRange(metricTagCounters.Select(counter => new TagCounterUI()
{
result.Add(new TagCounterUI
{
Tag = counter.Tag,
Count = counter.Count
});
}
Tag = counter.Tag,
Count = counter.Count
}));

return result;
}
Expand Down
4 changes: 2 additions & 2 deletions AppInspector/MetaData.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ public class MetaData
/// List of detected tag counters i.e. metrics
/// </summary>
[JsonProperty(PropertyName = "tagCounters")]
public List<MetricTagCounter> TagCounters { get; set; }
public ConcurrentStack<MetricTagCounter> TagCounters { get; set; }

/// <summary>
/// List of detailed MatchRecords from scan
Expand Down Expand Up @@ -225,7 +225,7 @@ public MetaData(string applicationName, string sourcePath)
};

Languages = new ConcurrentDictionary<string, int>();
TagCounters = new List<MetricTagCounter>();
TagCounters = new ConcurrentStack<MetricTagCounter>();
}

internal void IncrementFilesAnalyzed(int amount = 1)
Expand Down
99 changes: 42 additions & 57 deletions AppInspector/MetaDataHelper.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,54 +49,43 @@ public void AddMatchRecord(MatchRecord matchRecord)
}
}

// single standard properties we capture from any supported file type; others just captured as general tag matches...
if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Author")))
{
Metadata.Authors = ExtractValue(matchRecord.Sample);
}

if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Publisher")))
{
Metadata.Authors = ExtractValue(matchRecord.Sample);
}

if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Description")))
{
Metadata.Description = ExtractValue(matchRecord.Sample);
}

if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Name")))
{
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
}

if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Version")))
{
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
}

if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Target.Processor")))
{
_ = Metadata.CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
}

if (matchRecord.Tags.Any(v => v.Contains("Metadata.Application.Output.Type")))
{
_ = Metadata.Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
}

if (matchRecord.Tags.Any(v => v.Contains("Platform.OS")))
{
_ = Metadata.OSTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
}

if (matchRecord.Tags.Any(v => v.Contains("Metric.")))
//Update metric counters for default or user specified tags; don't add as match detail
foreach (var tag in matchRecord.Tags)
{
Metadata.TagCounters.Add(new MetricTagCounter()
switch (tag)
{
Tag = matchRecord.Tags[0],
Count = 0
});
case "Metadata.Application.Author":
case "Metadata.Application.Publisher":
Metadata.Authors = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Description":
Metadata.Description = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Name":
Metadata.ApplicationName = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Version":
Metadata.SourceVersion = ExtractValue(matchRecord.Sample);
break;
case "Metadata.Application.Target.Processor":
_ = Metadata.CPUTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Metadata.Application.Output.Type":
_ = Metadata.Outputs.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
case "Platform.OS":
_ = Metadata.OSTargets.TryAdd(ExtractValue(matchRecord.Sample).ToLower(), 0);
break;
default:
if (tag.Contains("Metric."))
{
Metadata.TagCounters.Push(new MetricTagCounter()
{
Tag = tag
});
}
break;
}
}

//safeguard sample output now that we've matched properties for blocking browser xss
Expand All @@ -109,20 +98,16 @@ public void AddMatchRecord(MatchRecord matchRecord)
_ = Metadata.AppTypes.TryAdd(solutionType,0);
}

//Update metric counters for default or user specified tags; don't add as match detail
bool counterOnlyTag = false;
foreach (MetricTagCounter counter in Metadata.TagCounters)
bool CounterOnlyTagSet = false;
var selected = Metadata.TagCounters.Where(x => matchRecord.Tags.Any(y => y.Contains(x.Tag)));
foreach (var select in selected)
{
if (matchRecord.Tags.Any(v => v.Contains(counter.Tag)))
{
counterOnlyTag = true;
counter.Count++;
break;
}
CounterOnlyTagSet = true;
select.IncrementCount();
}

//omit adding if only a counter metric tag
if (!counterOnlyTag)
//omit adding if ther a counter metric tag
if (!CounterOnlyTagSet)
{
//update list of unique tags as we go
foreach (string tag in matchRecord.Tags)
Expand Down
10 changes: 9 additions & 1 deletion AppInspector/MetricTagCounter.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
// Licensed under the MIT License.

using Newtonsoft.Json;
using System.Threading;

namespace Microsoft.ApplicationInspector.Commands
{
Expand All @@ -14,9 +15,16 @@ public class MetricTagCounter
public string Tag { get; set; }

[JsonProperty(PropertyName = "count")]
public int Count { get; set; }
public int Count { get { return _count; } }

[JsonProperty(PropertyName = "includeAsMatch")]
public bool IncludeAsMatch => false;

private int _count = 0;

internal void IncrementCount(int amount = 1)
{
Interlocked.Add(ref _count, amount);
}
}
}
1 change: 0 additions & 1 deletion MultiExtractor/Extractor.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,6 @@ private void CheckResourceGovernor(long additionalBytes = 0)
/// <returns>Extracted files</returns>
public IEnumerable<FileEntry> ExtractFile(string filename, bool parallel = false)
{
Console.Write(parallel);
if (!File.Exists(filename))
{
Logger.Warn("ExtractFile called, but {0} does not exist.", filename);
Expand Down

0 comments on commit fdc1f87

Please sign in to comment.