Merge pull request #3 from fossology/HastagAB/Developement

UnitTest and Automated Test Script.

.gitignore

@@ -6,6 +6,8 @@ __pycache__/
 # C extensions
 *.so
 
+TestFiles/
+
 # Distribution / packaging
 .Python
 build/

extractor/binder.py

@@ -79,10 +79,11 @@ def readMultiLineSame(file, syntax: str):
 def readMultiLineDiff(file, startSyntax: str, endSyntax: str):
     output, startLine, endLine = [], [], []
     content = ""
-    lines_of_comment = 0
+    total_lines, line_of_comments, blank_lines = 0,0,0
     copy = False
     with open(file) as f:
         for lineNumber, line in enumerate(f, start=1):
+            total_lines += 1
             if line.strip() == startSyntax:
                 copy = True
                 startLine.append(lineNumber)
@@ -92,12 +93,14 @@ def readMultiLineDiff(file, startSyntax: str, endSyntax: str):
                 content = ""
                 endLine.append(lineNumber)
             if copy:
-                lines_of_comment += 1
+                line_of_comments += 1
                 content = content + line.replace('\n','')
-        lines_of_comment += len(output)
+            if not line.strip():
+                blank_lines += 1
+        line_of_comments += len(output)
         output = [s.strip(startSyntax) for s in output]
         output = [s.strip(endSyntax) for s in output]
-    return startLine, endLine, output, lines_of_comment
+    return startLine, endLine, output, line_of_comments, total_lines, blank_lines
 
 
 class CommentSyntax:

extractor/languages/c.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def cExtractor(file):
     result = CommentSyntax()

extractor/languages/c_sharp.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def c_sharpExtractor(file):
     result = CommentSyntax()

extractor/languages/cpp.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def cppExtractor(file):
     result = CommentSyntax()

extractor/languages/css.py

@@ -20,10 +20,54 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def cssExtractor(file):
     output = CommentSyntax()
     result1 = output.slashStar(file)
+    file = file.split("/")
 
-    return result1
+    output = {
+        "metadata": [{
+        "filename": file[-1],
+        "lang": "CSS",
+        "total_lines": result1[4],
+        "total_lines_of_comments": result1[3],
+        "blank_lines": result1[5],
+        "sloc": result1[4]-(result1[3]+result1[5])
+        }],
+        "single_line_comment": [],
+        "multi_line_comment": []
+    }
+    if result1:
+        for idx,i in enumerate(result1[0]):
+            output['multi_line_comment'].append({"start_line": result1[0][idx], "end_line": result1[1][idx], "comment": result1[2][idx]})
+    return output
+
+
+def cssSource(file, newFile: str):
+    closingCount = 0
+    copy = True
+    with open(newFile, 'w+') as f1:
+        with open(file) as f:
+            for lineNumber, line in enumerate(f, start=1):
+                if line.strip() == '/*':
+                    closingCount+=1
+                    copy = False
+                    if closingCount%2 == 0:
+                        copy = True
+
+                if line.strip() == '*/':
+                    closingCount+=1
+                    copy = False
+                    if closingCount%2 == 0:
+                        copy = True
+
+                if copy:
+                    if line.strip() != '/*' and line.strip() != '*/':
+                        # Templine = line.replace(" ","")
+                        # if Templine[0:2] != "syntax":            # Syntax for single line comment
+                        f1.write(line)
+    f.close()
+    f1.close()
+    return newFile

extractor/languages/go.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def goExtractor(file):
     result = CommentSyntax()

extractor/languages/haskell.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def haskellExtractor(file):
     result = CommentSyntax()

extractor/languages/html.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def htmlExtractor(file):
     result = CommentSyntax()
@@ -31,10 +31,10 @@ def htmlExtractor(file):
         "metadata": [{
         "filename": file[-1],
         "lang": "HTML",
-        "total_lines": result1[1],
+        "total_lines": result1[4],
         "total_lines_of_comments": result1[3]+result2[3],
-        "blank_lines": result1[2],
-        "sloc": result1[1]-(result1[3]+result2[3]+result1[2])
+        "blank_lines": result1[5],
+        "sloc": result1[4]-(result1[3]+result2[3]+result1[5])
         }],
         "single_line_comment": [],
         "multi_line_comment": []

extractor/languages/java.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def javaExtractor(file):
     result = CommentSyntax()

extractor/languages/javascript.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def javascriptExtractor(file):
     result = CommentSyntax()

extractor/languages/kotlin.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def kotlinExtractor(file):
     result = CommentSyntax()

extractor/languages/matlab.py

@@ -20,12 +20,12 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def matlabExtractor(file):
     result = CommentSyntax()
-    result1 = result.percentageCurlybraces(file)
-    result2 = result.percentage(file)
+    result2 = result.percentageCurlybraces(file)
+    result1 = result.percentage(file)
     file = file.split("/")
     output = {
         "metadata": [{

extractor/languages/perl.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def perlExtractor(file):
     result = CommentSyntax()

extractor/languages/php.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def phpExtractor(file):
     result = CommentSyntax()

extractor/languages/python.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 
 def pythonExtractor(file):

extractor/languages/r.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def rExtractor(file):
     result = CommentSyntax()

extractor/languages/ruby.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def rubyExtractor(file):
     result = CommentSyntax()
@@ -51,7 +51,7 @@ def rubyExtractor(file):
     return output
 
 
-def crubySource(file, newFile: str):
+def rubySource(file, newFile: str):
     closingCount = 0
     copy = True
     with open(newFile, 'w+') as f1:

extractor/languages/rust.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def rustExtractor(file):
     result = CommentSyntax()

extractor/languages/scala.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def scalaExtractor(file):
     result = CommentSyntax()

extractor/languages/shell.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def shellExtractor(file):
     result = CommentSyntax()

extractor/languages/swift.py

@@ -20,7 +20,7 @@
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301  USA
 '''
 
-from extractor.binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
+from binder import readSingleLine, readMultiLineSame, readMultiLineDiff, CommentSyntax
 
 def swiftExtractor(file):
     result = CommentSyntax()

extractor/languages/tests/test_c.py

@@ -0,0 +1,58 @@
+import unittest
+import re, os
+from languages import c
+from binder import readSingleLine,readMultiLineDiff
+
+class CTest(unittest.TestCase):
+
+    def test_output(self):
+        path = os.path.join(os.getcwd(),"languages/tests/TestFiles/textcomment.c")
+        regex = r'''(\/\/\s*[\w #\.()@+-_*\d]*)'''
+        self.syntax_start = "/*"
+        self.syntax_end ='*/'
+        comment_single = c.readSingleLine(path,regex)
+        comment_multiline = c.readMultiLineDiff(path,self.syntax_start,self.syntax_end)
+
+        self.assertTrue(comment_single)
+        self.assertTrue(comment_multiline)
+
+
+
+    def test_outputFormat(self):
+        path = os.path.join(os.getcwd(),"languages/tests/TestFiles/textcomment.c")
+        regex = r'''(\/\/\s*[\w #\.()@+-_*\d]*)'''
+        self.syntax_start = "/*"
+        self.syntax_end = "*/"
+        expected = c.cExtractor(path)
+        comment_single = readSingleLine(path,regex)
+        comment_multiline = readMultiLineDiff(path,self.syntax_start,self.syntax_end)
+        file = path.split("/")
+        output = {
+        "metadata": [{
+        "filename": file[-1],
+        "lang": "C",
+        "total_lines": comment_single[1],
+        "total_lines_of_comments": comment_single[3]+comment_multiline[3],
+        "blank_lines": comment_single[2],
+        "sloc": comment_single[1]-(comment_single[3]+comment_multiline[3]+comment_single[2])
+        }],
+        "single_line_comment": [],
+        "multi_line_comment": []
+    }
+        if comment_single:
+            for i in comment_single[0]:
+                output['single_line_comment'].append({"line_number" :i[0],"comment": i[1]})
+
+        if comment_multiline:
+            for idx,i in enumerate(comment_multiline[0]):
+                output['multi_line_comment'].append({"start_line": comment_multiline[0][idx], "end_line": comment_multiline[1][idx], "comment": comment_multiline[2][idx]})
+
+        self.assertEqual(output,expected)  
+
+    def test_Source(self):
+        path = os.path.join(os.getcwd(),"languages/tests/TestFiles/textcomment.c")
+        name = "source.txt"
+        newfile = c.cSource(path,name)
+
+        self.assertTrue(newfile)
+