Filtered duplicates from queried urls in test

pixee · Jan 8, 2025 · 0cfe674 · 0cfe674
1 parent 923e14f
commit 0cfe674
Show file tree

Hide file tree

Showing 28 changed files with 40 additions and 34 deletions.
diff --git a/integration_tests/test_codemod_urls.py b/integration_tests/test_codemod_urls.py
@@ -37,9 +37,15 @@ async def check_accessible_urls(urls):
 
 @pytest.mark.asyncio
 async def test_codemod_reference_urls():
-    urls = [
-        ref.url for codemod in registry.codemods for ref in codemod._metadata.references
-    ]
+    urls = list(
+        set(
+            [
+                ref.url
+                for codemod in registry.codemods
+                for ref in codemod._metadata.references
+            ]
+        )
+    )
     await check_accessible_urls(urls)
 
 

diff --git a/src/core_codemods/add_requests_timeouts.py b/src/core_codemods/add_requests_timeouts.py
@@ -27,7 +27,7 @@ def on_result_found(self, original_node, updated_node):
             Reference(
                 url="https://docs.python-requests.org/en/master/user/quickstart/#timeouts"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/1088.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/1088"),
         ],
     ),
     detector=SemgrepRuleDetector(

diff --git a/src/core_codemods/defectdojo/semgrep/avoid_insecure_deserialization.py b/src/core_codemods/defectdojo/semgrep/avoid_insecure_deserialization.py
@@ -58,7 +58,7 @@ def leave_Call(
             ],
         ),
         references=[
-            Reference(url="https://cwe.mitre.org/data/definitions/502.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/502"),
         ],
     ),
     transformer=LibcstTransformerPipeline(

diff --git a/src/core_codemods/defectdojo/semgrep/django_secure_set_cookie.py b/src/core_codemods/defectdojo/semgrep/django_secure_set_cookie.py
@@ -52,7 +52,7 @@ def leave_Call(self, original_node: cst.Call, updated_node: cst.Call) -> cst.Cal
             ],
         ),
         references=[
-            Reference(url="https://cwe.mitre.org/data/definitions/614.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/614"),
         ],
     ),
     transformer=LibcstTransformerPipeline(DjangoSecureSetCookieTransformer),

diff --git a/src/core_codemods/disable_graphql_introspection.py b/src/core_codemods/disable_graphql_introspection.py
@@ -128,10 +128,10 @@ def _is_introspection_rule_or_starred(
                 url="https://owasp.org/www-project-web-security-testing-guide/v42/4-Web_Application_Security_Testing/12-API_Testing/01-Testing_GraphQL#introspection-queries",
             ),
             Reference(
-                url="https://cwe.mitre.org/data/definitions/200.html",
+                url="https://cwe.mitre.org/data/definitions/200",
             ),
             Reference(
-                url="https://cwe.mitre.org/data/definitions/669.html",
+                url="https://cwe.mitre.org/data/definitions/669",
             ),
         ],
     ),

diff --git a/src/core_codemods/django_debug_flag_on.py b/src/core_codemods/django_debug_flag_on.py
@@ -16,7 +16,7 @@ class DjangoDebugFlagOn(SimpleCodemod):
             Reference(
                 url="https://docs.djangoproject.com/en/4.2/ref/settings/#std-setting-DEBUG"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/489.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/489"),
         ],
     )
     change_description = "Flip `Django` debug flag to off."

diff --git a/src/core_codemods/django_session_cookie_secure_off.py b/src/core_codemods/django_session_cookie_secure_off.py
@@ -16,7 +16,7 @@ class DjangoSessionCookieSecureOff(SimpleCodemod):
             Reference(
                 url="https://docs.djangoproject.com/en/4.2/ref/settings/#session-cookie-secure"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/614.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/614"),
         ],
     )
     change_description = "Sets Django's `SESSION_COOKIE_SECURE` flag if off or missing."

diff --git a/src/core_codemods/file_resource_leak.py b/src/core_codemods/file_resource_leak.py
@@ -73,8 +73,8 @@ def line_filter(x):
         summary="Automatically Close Resources",
         review_guidance=ReviewGuidance.MERGE_WITHOUT_REVIEW,
         references=[
-            Reference(url="https://cwe.mitre.org/data/definitions/404.html"),
-            Reference(url="https://cwe.mitre.org/data/definitions/772.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/404"),
+            Reference(url="https://cwe.mitre.org/data/definitions/772"),
         ],
     ),
     transformer=LibcstTransformerPipeline(FileResourceLeakTransformer),

diff --git a/src/core_codemods/flask_enable_csrf_protection.py b/src/core_codemods/flask_enable_csrf_protection.py
@@ -19,7 +19,7 @@ class FlaskEnableCSRFProtection(
         references=[
             Reference(url="https://owasp.org/www-community/attacks/csrf"),
             Reference(url="https://flask-wtf.readthedocs.io/en/1.2.x/csrf/"),
-            Reference(url="https://cwe.mitre.org/data/definitions/352.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/352"),
         ],
     )
 

diff --git a/src/core_codemods/harden_pickle_load.py b/src/core_codemods/harden_pickle_load.py
@@ -22,7 +22,7 @@ class HardenPickleLoad(SimpleCodemod, ImportModifierCodemod):
                 url="https://github.com/trailofbits/fickling",
             ),
             Reference(
-                url="https://cwe.mitre.org/data/definitions/502.html",
+                url="https://cwe.mitre.org/data/definitions/502",
             ),
         ],
     )

diff --git a/src/core_codemods/harden_pyyaml.py b/src/core_codemods/harden_pyyaml.py
@@ -126,7 +126,7 @@ def _update_bases(
             Reference(
                 url="https://github.com/yaml/pyyaml/wiki/PyYAML-yaml.load(input)-Deprecation"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/502.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/502"),
         ],
     ),
     detector=SemgrepRuleDetector(

diff --git a/src/core_codemods/harden_ruamel.py b/src/core_codemods/harden_ruamel.py
@@ -11,7 +11,7 @@ class HardenRuamel(SimpleCodemod):
             Reference(
                 url="https://owasp.org/www-community/vulnerabilities/Deserialization_of_untrusted_data"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/502.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/502"),
         ],
     )
     change_description = (

diff --git a/src/core_codemods/https_connection.py b/src/core_codemods/https_connection.py
@@ -59,7 +59,7 @@ class HTTPSConnection(SimpleCodemod):
             Reference(
                 url="https://urllib3.readthedocs.io/en/stable/reference/urllib3.connectionpool.html#urllib3.HTTPConnectionPool"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/319.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/319"),
         ],
     )
 

diff --git a/src/core_codemods/limit_readline.py b/src/core_codemods/limit_readline.py
@@ -11,7 +11,7 @@ class LimitReadline(SimpleCodemod):
         summary="Limit readline()",
         review_guidance=ReviewGuidance.MERGE_AFTER_CURSORY_REVIEW,
         references=[
-            Reference(url="https://cwe.mitre.org/data/definitions/400.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/400"),
         ],
     )
     change_description = "Adds a size limit argument to readline() calls."

diff --git a/src/core_codemods/lxml_safe_parser_defaults.py b/src/core_codemods/lxml_safe_parser_defaults.py
@@ -17,7 +17,7 @@ class LxmlSafeParserDefaults(SimpleCodemod):
             Reference(
                 url="https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/611.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/611"),
         ],
     )
     change_description = "Replace `lxml` parser parameters with safe defaults."

diff --git a/src/core_codemods/lxml_safe_parsing.py b/src/core_codemods/lxml_safe_parsing.py
@@ -17,7 +17,7 @@ class LxmlSafeParsing(SimpleCodemod):
             Reference(
                 url="https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/611.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/611"),
         ],
     )
     change_description = (

diff --git a/src/core_codemods/replace_flask_send_file.py b/src/core_codemods/replace_flask_send_file.py
@@ -18,7 +18,7 @@ class ReplaceFlaskSendFile(SimpleCodemod, NameAndAncestorResolutionMixin):
                 url="https://flask.palletsprojects.com/en/3.0.x/api/#flask.send_from_directory"
             ),
             Reference(url="https://owasp.org/www-community/attacks/Path_Traversal"),
-            Reference(url="https://cwe.mitre.org/data/definitions/35.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/35"),
         ],
     )
 

diff --git a/src/core_codemods/requests_verify.py b/src/core_codemods/requests_verify.py
@@ -13,7 +13,7 @@ class RequestsVerify(SimpleCodemod):
             Reference(
                 url="https://owasp.org/www-community/attacks/Manipulator-in-the-middle_attack"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/295.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/295"),
         ],
     )
     change_description = (

diff --git a/src/core_codemods/secure_flask_cookie.py b/src/core_codemods/secure_flask_cookie.py
@@ -14,7 +14,7 @@ class SecureFlaskCookie(SimpleCodemod, SecureCookieMixin):
             Reference(
                 url="https://owasp.org/www-community/controls/SecureCookieAttribute"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/614.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/614"),
         ],
     )
     change_description = "Flask response `set_cookie` call should be called with `secure=True`, `httponly=True`, and `samesite='Lax'`."

diff --git a/src/core_codemods/secure_flask_session_config.py b/src/core_codemods/secure_flask_session_config.py
@@ -23,9 +23,9 @@ class SecureFlaskSessionConfig(SimpleCodemod, Codemod):
             Reference(
                 url="https://cheatsheetseries.owasp.org/cheatsheets/Session_Management_Cheat_Sheet.html"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/319.html"),
-            Reference(url="https://cwe.mitre.org/data/definitions/352.html"),
-            Reference(url="https://cwe.mitre.org/data/definitions/614.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/319"),
+            Reference(url="https://cwe.mitre.org/data/definitions/352"),
+            Reference(url="https://cwe.mitre.org/data/definitions/614"),
         ],
     )
     change_description = "Flip Flask session configuration if defined as insecure."

diff --git a/src/core_codemods/semgrep/semgrep_nan_injection.py b/src/core_codemods/semgrep/semgrep_nan_injection.py
@@ -126,7 +126,7 @@ def visit_Call(self, node: cst.Call) -> None:
             ],
         ),
         references=[
-            Reference(url="https://cwe.mitre.org/data/definitions/704.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/704"),
         ],
     ),
     transformer=LibcstTransformerPipeline(NanInjectionTransformer),

diff --git a/src/core_codemods/semgrep/semgrep_no_csrf_exempt.py b/src/core_codemods/semgrep/semgrep_no_csrf_exempt.py
@@ -55,7 +55,7 @@ def leave_Decorator(
             ],
         ),
         references=[
-            Reference(url="https://cwe.mitre.org/data/definitions/352.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/352"),
         ],
     ),
     transformer=LibcstTransformerPipeline(RemoveCsrfExemptTransformer),

diff --git a/src/core_codemods/semgrep/semgrep_rsa_key_size.py b/src/core_codemods/semgrep/semgrep_rsa_key_size.py
@@ -76,7 +76,7 @@ def match_location(self, pos, result):
             ],
         ),
         references=[
-            Reference(url="https://cwe.mitre.org/data/definitions/326.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/326"),
         ],
     ),
     transformer=LibcstTransformerPipeline(RsaKeySizeTransformer),

diff --git a/src/core_codemods/sql_parameterization.py b/src/core_codemods/sql_parameterization.py
@@ -370,7 +370,7 @@ def _remove_literal_and_gather_extra(
         review_guidance=ReviewGuidance.MERGE_AFTER_CURSORY_REVIEW,
         references=[
             Reference(url="https://cwe.mitre.org/data/definitions/20"),
-            Reference(url="https://cwe.mitre.org/data/definitions/89.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/89"),
             Reference(url="https://owasp.org/www-community/attacks/SQL_Injection"),
         ],
     ),

diff --git a/src/core_codemods/subprocess_shell_false.py b/src/core_codemods/subprocess_shell_false.py
@@ -79,7 +79,7 @@ def first_arg_is_not_string(self, original_node: cst.Call) -> bool:
                 url="https://en.wikipedia.org/wiki/Code_injection#Shell_injection"
             ),
             Reference(url="https://stackoverflow.com/a/3172488"),
-            Reference(url="https://cwe.mitre.org/data/definitions/78.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/78"),
         ],
     ),
     transformer=LibcstTransformerPipeline(SubprocessShellFalseTransformer),

diff --git a/src/core_codemods/upgrade_sslcontext_minimum_version.py b/src/core_codemods/upgrade_sslcontext_minimum_version.py
@@ -13,7 +13,7 @@ class UpgradeSSLContextMinimumVersion(SimpleCodemod, NameResolutionMixin):
             ),
             Reference(url="https://datatracker.ietf.org/doc/rfc8996/"),
             Reference(url="https://www.digicert.com/blog/depreciating-tls-1-0-and-1-1"),
-            Reference(url="https://cwe.mitre.org/data/definitions/326.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/326"),
         ],
     )
     change_description = "Replaces minimum SSL/TLS version for SSLContext."

diff --git a/src/core_codemods/upgrade_sslcontext_tls.py b/src/core_codemods/upgrade_sslcontext_tls.py
@@ -13,7 +13,7 @@ class UpgradeSSLContextTLS(SimpleCodemod):
             ),
             Reference(url="https://datatracker.ietf.org/doc/rfc8996/"),
             Reference(url="https://www.digicert.com/blog/depreciating-tls-1-0-and-1-1"),
-            Reference(url="https://cwe.mitre.org/data/definitions/326.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/326"),
         ],
     )
     change_description = "Replaces known insecure TLS/SSL protocol versions in SSLContext with secure ones."

diff --git a/src/core_codemods/use_defused_xml.py b/src/core_codemods/use_defused_xml.py
@@ -51,7 +51,7 @@ def dependency(self) -> Dependency:
             Reference(
                 url="https://cheatsheetseries.owasp.org/cheatsheets/XML_External_Entity_Prevention_Cheat_Sheet.html"
             ),
-            Reference(url="https://cwe.mitre.org/data/definitions/611.html"),
+            Reference(url="https://cwe.mitre.org/data/definitions/611"),
         ],
     ),
     transformer=LibcstTransformerPipeline(UseDefusedXmlTransformer),