RootExclusionFix (#212)

* Fix #207

* Fix #210 to prevent root source with name that would exclude from scan

* Filter test change correction

AppInspector/Commands/AnalyzeCommand.cs

@@ -612,26 +612,24 @@ private string ExtractDependency(string text, int startIndex, SearchPattern patt
 
         #endregion ProcessingAssist
 
-        private void UnZipAndProcess(string filename, ArchiveFileType archiveFileType, bool topLevel = true)
+        private void UnZipAndProcess(string filePath, ArchiveFileType archiveFileType, bool topLevel = true)
         {
             // zip itself may be in excluded list i.e. sample, test or similar unless ignore filter requested
-            if (_fileExclusionList != null && _fileExclusionList.Any(v => filename.ToLower().Contains(v)))
+            if (ExcludeFileFromScan(filePath))
             {
-                WriteOnce.SafeLog(MsgHelp.FormatString(MsgHelp.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filename), LogLevel.Warn);
-                _metaDataHelper.Metadata.IncrementFilesSkipped();
                 return;
             }
 
             //zip itself may be too huge for timely processing
-            if (new FileInfo(filename).Length > WARN_ZIP_FILE_SIZE)
+            if (new FileInfo(filePath).Length > WARN_ZIP_FILE_SIZE)
             {
                 if (topLevel)
                 {
                     WriteOnce.General(MsgHelp.GetString(MsgHelp.ID.ANALYZE_COMPRESSED_FILESIZE_WARN));
                 }
                 else
                 {
-                    WriteOnce.SafeLog("Decompressing large file " + filename, LogLevel.Warn);
+                    WriteOnce.SafeLog("Decompressing large file " + filePath, LogLevel.Warn);
                 }
             }
             else
@@ -642,16 +640,16 @@ private void UnZipAndProcess(string filename, ArchiveFileType archiveFileType, b
                 }
                 else
                 {
-                    WriteOnce.SafeLog("Decompressing file " + filename, LogLevel.Warn);
+                    WriteOnce.SafeLog("Decompressing file " + filePath, LogLevel.Warn);
                 }
             }
 
-            LastUpdated = File.GetLastWriteTime(filename);
+            LastUpdated = File.GetLastWriteTime(filePath);
             _metaDataHelper.Metadata.PackageTypes.Add(MsgHelp.GetString(MsgHelp.ID.ANALYZE_COMPRESSED_FILETYPE));
 
             try
             {
-                IEnumerable<FileEntry> files = Extractor.ExtractFile(filename).Where(x => x != null);
+                IEnumerable<FileEntry> files = Extractor.ExtractFile(filePath).Where(x => x != null);
 
                 if (_options.SingleThread)
                 {
@@ -699,7 +697,7 @@ private void UnZipAndProcess(string filename, ArchiveFileType archiveFileType, b
             }
             catch (Exception)
             {
-                string errmsg = MsgHelp.FormatString(MsgHelp.ID.ANALYZE_COMPRESSED_ERROR, filename);
+                string errmsg = MsgHelp.FormatString(MsgHelp.ID.ANALYZE_COMPRESSED_ERROR, filePath);
                 WriteOnce.Error(errmsg);
                 throw;
             }
@@ -726,11 +724,9 @@ private bool FileChecksPassed(string filePath, ref LanguageInfo languageInfo, lo
 
             _metaDataHelper.AddLanguage(languageInfo.Name);
 
-            // 2. Skip excluded files i.e. sample, test or similar unless ignore filter requested
-            if (_fileExclusionList != null && _fileExclusionList.Any(v => filePath.ToLower().Contains(v)))
+            // 2. Check for exclusions
+            if (ExcludeFileFromScan(filePath))
             {
-                WriteOnce.SafeLog(MsgHelp.FormatString(MsgHelp.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filePath), LogLevel.Warn);
-                _metaDataHelper.Metadata.IncrementFilesSkipped();
                 return false;
             }
 
@@ -753,5 +749,30 @@ private bool FileChecksPassed(string filePath, ref LanguageInfo languageInfo, lo
 
             return true;
         }
+
+
+        /// <summary>
+        /// Allow callers to exclude files that are not core code files and may otherwise report false positives for matches
+        /// Does not apply to root scan folder which may be named .\test etc. but to subdirectories only
+        /// </summary>
+        /// <param name="filePath"></param>
+        /// <returns></returns>
+        private bool ExcludeFileFromScan(string filePath)
+        {
+            string rootScanDirectory = Directory.Exists(_options.SourcePath) ? _options.SourcePath : Path.GetDirectoryName(_options.SourcePath);
+            bool scanningRootFolder = Path.GetDirectoryName(filePath).ToLower() == rootScanDirectory.ToLower();
+            // 2. Skip excluded files i.e. sample, test or similar from sub-directories (not root #210) unless ignore filter requested
+            if (!scanningRootFolder)
+            {
+                if (_fileExclusionList != null && _fileExclusionList.Any(v => filePath.ToLower().Contains(v)))
+                {
+                    WriteOnce.SafeLog(MsgHelp.FormatString(MsgHelp.ID.ANALYZE_EXCLUDED_TYPE_SKIPPED, filePath), LogLevel.Warn);
+                    _metaDataHelper.Metadata.IncrementFilesSkipped();
+                    return true;
+                }
+            }
+
+            return false;
+        }
     }
 }

UnitTest.Commands/Tests_CLI/TestAnalyzeCmd.cs

@@ -344,13 +344,13 @@ public void DefaultAndCustomRulesPosMatches_Pass()
         }
 
         [TestMethod]
-        public void ExclusionFilter_Fail()
+        public void ExclusionFilter_Pass()
         {
             AnalyzeResult.ExitCode exitCode = AnalyzeResult.ExitCode.CriticalError;
             try
             {
                 string args = string.Format(@"analyze -s {0} -r {1} -f json -o {2}",
-                    Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\simple\main.cpp"),
+                    Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\project\one"),
                     Path.Combine(Helper.GetPath(Helper.AppPath.testRules), @"myrule.json"),
                     Path.Combine(Helper.GetPath(Helper.AppPath.testOutput), @"output.txt"));
 

UnitTest.Commands/Tests_NuGet/TestAnalyzeCmd.cs

@@ -265,12 +265,11 @@ public void DefaultAndCustomRulesMatched_Pass()
         }
 
         [TestMethod]
-        public void ExclusionFilter_Fail()
+        public void ExclusionFilter_Pass()
         {
             AnalyzeOptions options = new AnalyzeOptions()
             {
-                SourcePath = Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\simple\main.cpp")
-                //FilePathExclusions = "none", //allow source under unittest path
+                SourcePath = Path.Combine(Helper.GetPath(Helper.AppPath.testSource), @"unzipped\project\one")
             };
 
             AnalyzeResult.ExitCode exitCode = AnalyzeResult.ExitCode.CriticalError;

UnitTest.Commands/source/unzipped/project/one/test/main.cpp

@@ -0,0 +1,181 @@
+MD5
+include <system.h>
+include <assert.h>
+include "someheader.h"
+include "someheader2.h"
+
+SECURITY_FLAG_CA
+
+SECURITY_FLAG_IGNORE_CA
+
+<OutputType>exe< / OutputType>
+
+Windows
+
+Custom1
+
+windows
+
+android
+
+linux
+
+certificate
+
+sharedAccessPolicy
+
+class SSLv3
+
+	class
+	try
+
+	sleep(d);
+
+irs
+
+salary
+
+bank
+
+creditcard
+
+ssnumber
+
+Apache
+
+login b
+
+RegSetValueEx("foo");
+
+int main(int argc)
+{
+	JsonConvert.DeserializeObject<List<FileTagGroup>>(File.ReadAllText(filePath));
+
+	libtiff
+
+		File.Delete
+
+		SQL
+
+		Select* from table
+
+		mutex
+
+		MFC
+
+		WCF
+
+		Word
+
+		microsoft
+
+		Office
+
+		Word  Excel
+
+		Mocking
+
+		xml.load
+
+		json deserialize
+
+		flash
+
+		Roles w;
+
+	<OutputType>Exe< / OutputType>
+
+		<ConfigurationType>DynamicLibrary< / ConfigurationType>
+
+		// Win64
+
+		android
+
+		azure
+
+		Windows
+
+		char src[40];
+	char dest[100];
+
+	Deserialize e;
+	unserialize("something");
+
+	Net::HTTP k;
+
+	socket f
+
+		webclient w;
+
+	fwrite  k;
+
+	fwrite l;
+
+	fwrite j;
+
+	eval(src)
+
+		socket j;
+
+	ajax.post
+
+		webclient
+
+		paypal;
+
+	salary;
+
+	ssnumber;
+
+	facebook;
+
+	class foo;
+
+	//high risk
+
+	//high risk
+
+	//TODO
+	TODO
+
+		class x;
+	class y;
+	function w;
+
+	form tt;
+	form gg;
+
+	try
+	{
+	}
+	exception()
+	{
+		sleep(4);
+	}
+
+	//mcrypt d;
+	//AES d;
+
+	SSLv3  D;
+	TLSv1  d;
+
+	log ff;
+	log
+
+		MFC
+
+		//DESCryptoServiceProvider x = new DESCryptoServiceProvider();
+		//DES y; //works
+		//DESEngine eng;
+		//TODO.*crypt
+		//InitializeSecurityContext z;
+		MD5 hh;
+
+	MD2 TT;
+
+	MVC l;
+
+	socket p
+
+		digest
+}