ISSUE #12: Support schemas on the WEB

.github/workflows/self-test.yaml

@@ -30,4 +30,15 @@ jobs:
         with:
           json-schema-file: test/schema/json_schema.json
           yaml-json-file-dir: test/emptyJSONs,test/emptyYAMLSs
-          schema-mapping: "test/emptyJSONs:test/schema/json_schema.json,test/emptyYAMLSs:test/schema/json_schema.json"
+          schema-mapping: "test/emptyJSONs->test/schema/json_schema.json,test/emptyYAMLSs->test/schema/json_schema.json"
+      - name: Run Action (check online schema + mapping)
+        uses: ./
+        with:
+          json-schema-file: https://json-schema.org/draft-04/schema
+          yaml-json-file-dir: test/schema/json_schema.json
+          schema-mapping: "test/schema/json_schema.json->https://json-schema.org/draft-04/schema"
+      - name: Run Action (check online schema)
+        uses: ./
+        with:
+          json-schema-file: https://json-schema.org/draft-04/schema
+          yaml-json-file-dir: test/schema/json_schema.json

README.md

@@ -19,13 +19,13 @@ jobs:
         uses: lyubick/action-YAML-schema-validator@v2
         with:
           json-schema-file: path/to/my/cool/schema.json
-          yaml-json-file-dir: path/to/my/cool/yaml/file.yaml
+          yaml-json-file-dir: path/to/my/cool/yaml/file.yaml,path/to/my/cool/yaml/another/file.yaml
           recursive: false
           ignore-empty: false
-          schema-mapping: 'path/to/my/cool/yaml/file.yaml:path/to/my/cool/schema.json'
+          schema-mapping: 'path/to/my/cool/yaml/file.yaml->path/to/my/cool/schema.json,path/to/my/cool/yaml/another/file.yaml->https://path/to/schema'
 ```
 One should provide two parameters:
-- `json-schema-file`, required, points to legit JSON Schema files. In case of mapping this schema will be used as default (fallback) schema.
+- `json-schema-file`, required, points to legit JSON Schema files. In case of mapping this schema will be used as default (fallback) schema. Can point to online schemas that can be obtained via http(s), without authorization.
 - `yaml-json-file-dir`, is a comma separated list that contains:
   - Single YAML or JSON files
   - Directories that will be parsed for `.yaml` or `.yml` or `.json` files

action.yaml

@@ -8,8 +8,8 @@ branding:
 inputs:
   json-schema-file:
     description: |
-      JSON Schema file to validate against. In case mapping is provided, schema
-      provided here will be used as a default (fallback) schema.
+      JSON Schema file or HTTPS URL to file to validate against. In case, mapping
+      is provided, schema provided here will be used as a default schema.
     required: true
   yaml-json-file-dir:
     description: |
@@ -30,9 +30,8 @@ inputs:
     default: 'true'
   schema-mapping:
     description: |
-      File/Directory and Schema mapping, provided in a one of formats:
-      1. /path/to/file:/path/to/schema.json
-      2. /path/to/dir/:/path/to/schema.json
+      Files or directories with schema mapping, mapped using `->`. Multiple
+      mappings can be defined separating each mapping by comma.
     required: false
     default: ''
 runs:

test/schema/json_invalid_schema.json

@@ -0,0 +1,3 @@
+{
+  "title": 12345
+}

test/test_sanity.py

@@ -127,9 +127,55 @@ def test10_validate_file_with_schema_map(self):
             default_schema_path=f'{self.abs_path}/schema/json_schema.json'
         )
         files = validator.get_testing_filenames(f'{self.abs_path}/emptyJSONs/empty.json', False, False)
-        files_schemas = validator.get_filenames_with_schema(files, schemas, f"{self.abs_path}/emptyJSONs/empty.json:{self.abs_path}/schema/json_schema.json")
+        files_schemas = validator.get_filenames_with_schema(
+            files,
+            schemas,
+            f"{self.abs_path}/emptyJSONs/empty.json->{self.abs_path}/schema/json_schema.json"
+        )
         try:
             validator.validate_files(files_schemas)
             assert False
         except json.JSONDecodeError as exc:
             assert True
+
+    def test11_validate_http_schema(self):
+        schemas = validator.get_all_schemas(
+            schema_file_path=f'https://json-schema.org/draft-04/schema',
+            default_schema_path=f'https://json-schema.org/draft-04/schema'
+        )
+
+        files = validator.get_testing_filenames(f'{self.abs_path}/schema/json_schema.json', False, False)
+        files_schemas = validator.get_filenames_with_schema(
+            files,
+            schemas,
+            f"{self.abs_path}/schema/json_schema.json->https://json-schema.org/draft-04/schema"
+        )
+        validator.validate_files(files_schemas)
+        assert True
+
+        files = validator.get_testing_filenames(f'{self.abs_path}/schema/json_schema.json', False, False)
+        files_schemas = validator.get_filenames_with_schema(files, schemas, None)
+        validator.validate_files(files_schemas)
+        assert True
+
+        files = validator.get_testing_filenames(f'{self.abs_path}/schema/json_invalid_schema.json', False, False)
+        files_schemas = validator.get_filenames_with_schema(
+            files,
+            schemas,
+            f"{self.abs_path}/schema/json_invalid_schema.json->https://json-schema.org/draft-04/schema"
+        )
+
+        try:
+            validator.validate_files(files_schemas)
+            assert False
+        except Exception as exc:
+            assert True
+
+        files = validator.get_testing_filenames(f'{self.abs_path}/schema/', False, False)
+        files_schemas = validator.get_filenames_with_schema(files, schemas, None)
+
+        try:
+            validator.validate_files(files_schemas)
+            assert False
+        except Exception as exc:
+            assert True

validator.py

@@ -4,15 +4,22 @@
 import pathlib
 import re
 import sys
+from urllib.request import Request, urlopen
 from typing import List, Tuple, Optional
 
 import yaml
 from jsonschema import validate
 from jsonschema.exceptions import ValidationError
 
 
-def get_all_filenames(input_path: str, endings: List[str], is_recursive: bool) -> List[str]:
-    paths = list(input_path.split(','))
+SCHEMA_MAPPING_SEPARATOR = '->'
+
+
+def get_all_filenames(input_path: str or List[str], endings: List[str], is_recursive: bool) -> List[str]:
+    if isinstance(input_path, str):
+        paths = list(input_path.split(','))
+    else:
+        paths = input_path
 
     regex_endings = f'.*\\.({"|".join(endings)})'
 
@@ -40,10 +47,35 @@ def get_all_filenames(input_path: str, endings: List[str], is_recursive: bool) -
     return output_files
 
 
+def get_json_from_lines(lines: List[str]) -> dict:
+    return json.loads('\n'.join(lines))
+
+
+def handle_http_schema(input_url: str):
+    url = Request(input_url, headers={'User-Agent': 'Mozilla/5.0'})
+
+    content = urlopen(url).read().decode('utf-8')
+
+    return input_url, get_json_from_lines(list(content.split('\n')))
+
+
 def get_all_schemas(schema_file_path: str, default_schema_path: str) -> dict[str, dict]:
-    schema_files = get_all_filenames(schema_file_path, endings=['json'], is_recursive=False)
-    schemas = list(map(lambda x: (x, json.loads(''.join(open(x, 'r').readlines()))), schema_files))
-    schemas.append(('default', json.loads(''.join(open(default_schema_path, 'r').readlines()))))
+    split_path = list(schema_file_path.split(','))
+    http_path = list(filter(lambda x: 'https://' in x, split_path))
+    file_paths = list(filter(lambda x: 'https://' not in x, split_path))
+
+    schema_files = get_all_filenames(file_paths, endings=['json'], is_recursive=False)
+    schemas = list(map(lambda x: (x, get_json_from_lines(open(x, 'r').readlines())), schema_files))
+
+    if 'https://' in default_schema_path:
+        schemas.append(('default', handle_http_schema(default_schema_path)[1]))
+    else:
+        schemas.append(('default', get_json_from_lines(open(default_schema_path, 'r').readlines())))
+
+    if http_path:
+        http_schemas = list(map(lambda x: handle_http_schema(x), http_path))
+        schemas.extend(http_schemas)
+
     return dict(schemas)
 
 
@@ -60,7 +92,6 @@ def get_filenames_with_schema(
         test_files: List[str],
         schemas: dict[str, dict],
         mapping_str: Optional[str]) -> List[Tuple[str, dict]]:
-
     def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, dict]:
         for s in schema_map.keys():
             if filename in get_all_filenames(input_path=s, endings=['yaml', 'json', 'yml'], is_recursive=True):
@@ -69,7 +100,7 @@ def map_schema(filename: str, schema_map: dict[str, str]) -> Tuple[str, dict]:
         return filename, schemas['default']
 
     if mapping_str:
-        mapping = dict(list(map(lambda x: tuple_split(x, ':'), list(mapping_str.split(',')))))
+        mapping = dict(list(map(lambda x: tuple_split(x, SCHEMA_MAPPING_SEPARATOR), list(mapping_str.split(',')))))
         files_schema = list(map(lambda x: map_schema(x, mapping), test_files))
     else:
         files_schema = list(map(lambda x: (x, schemas['default']), test_files))
@@ -126,7 +157,7 @@ def tuple_split(inp: str, separator: str) -> Tuple[str, str]:
     input_mapped_schemas = ''
     if input_mapping:
         logging.error(input_mapping)
-        input_mapped_schemas = ','.join(list(map(lambda x: tuple_split(x, ':')[1], input_mapping.split(','))))
+        input_mapped_schemas = ','.join(list(map(lambda x: tuple_split(x, SCHEMA_MAPPING_SEPARATOR)[1], input_mapping.split(','))))
 
     input_schemas = get_all_schemas(schema_file_path=input_mapped_schemas, default_schema_path=args[0])