diff --git a/.gitmodules b/.gitmodules index d159c254..6179065b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "support/schematron"] - path = support/schematron - url = https://github.com/usnistgov/schematron.git - branch = master [submodule "support/xspec"] path = support/xspec url = https://github.com/AirQuick/xspec.git @@ -10,3 +6,6 @@ path = support/metaschema url = https://github.com/nikitawootten-nist/metaschema.git fetchRecurseSubmodules = false +[submodule "support/schxslt"] + path = support/schxslt + url = https://github.com/schxslt/schxslt.git diff --git a/bin/metaschema-xslt b/bin/metaschema-xslt index 06e3689b..0ac0be42 100755 --- a/bin/metaschema-xslt +++ b/bin/metaschema-xslt @@ -19,6 +19,8 @@ Subcommands: Generate JSON and XML schemas for a given metaschema - converter-gen METASCHEMA_XML OUTPUT_DIR SCHEMA_NAME [ADDITIONAL_ARGS] Generate converter transformations (XSLT) between JSON and XML for a given metaschema +- composition-validate METASCHEMA_XML [ADDITIONAL_ARGS] + Test metaschema composition (assembling modules and linking definitions) and report issues EOF } @@ -55,6 +57,9 @@ case "$SUBCOMMAND" in converter-gen) "$SRC_DIR"/converter-gen/mvn-converters-xpl.sh "$@" ;; + composition-validate) + "$SRC_DIR"/validate/mvn-composition-validate-xpl.sh "$@" + ;; ?) msg "Unknown subcommand: $SUBCOMMAND" exit 1 diff --git a/src/README.md b/src/README.md index e1ff5efa..4948fc1c 100644 --- a/src/README.md +++ b/src/README.md @@ -1,121 +1,64 @@ -# XSLT-M4 +# XSLT-M4 `src` -An XSLT implementation of the metaschema toolchain for generating schemas, converters, and model documentation. +An XSLT implementation of the [Metaschema](https://pages.nist.gov/metaschema) toolchain for generating schemas, converters, and model documentation. Typically any of these operations will combine several lower-level operations in a defined sequence. More details (produced by surveying the files) can be seen in [file-manifest.md](file-manifest.md). Note however that this file is not reliable if it is not more recent than the files described. -In addition to this readme, this folder contains XSLT transformations (`*.xsl`), and XProc pipelines (`xpr`). The XSLT provides stable runtimes to the supported operations as described below. The XProc The XProc is provided for convenience in development and debugging, and can be expected to change (develop/proliferate) somewhat more freely. These are currently XProc 1.0 pending further development. +In addition to this readme, this folder contains XSLT transformations (`*.xsl`), and XProc pipelines (`xpl`). The XSLT provides stable runtimes to the supported operations as described below. The XProc provides optimized runtimes when producing multiple outputs (results) from single inputs. -Routines described below provide the XSLT entry point for the service. In general, any service consumes a metaschema input and produces one or more outputs, publishable as artifacts. For any XSLT, an analogous XProc is usually discernable from a file name (for example, `make-metaschema-xsd.xpr` instantiates the same pipeline as `nist-metaschema-MAKE-XSD.xsl`). For documentation production, only XProc is given. +See each subdirectory README for more instructions. -## Generate schemas -### Generate XML Schema (XSD) +## common -source: metaschema (main module) +XSLT and logic used as common modules by other utilities. -XSLT: `nist-metaschema-MAKE-XSD.xsl` +Moving or removing this directory will often break things. -result: XSD (suffix `*.xsd`) +## compose -parameters: none +Implements a metaschema composition pipeline - producing a unified single metaschema from a metaschema top-level module, by performing imports and linking references. -The output is an XSD (XML Schema Definition) that can be used to provide structural and datatype validation over XML data instances, testing conformance to models defined by the metaschema. +This subroutine is a dependency for most other metaschema processes, so like `common` this directory should be kept in place. -### Generate JSON Schema +## converter-gen -source: metaschema (main module) +Logic to generate converter transformations (XSLT) capable of producing JSON from XML or XML from JSON, according to mappings defined by appropriate metaschema definitions, defining schemas to which the respective data sets are valid. -XSLT: `nist-metaschema-MAKE-JSON-SCHEMA.xsl` +## document -result: JSON Schema (suffix `*.json`) +Logic to create HTML-based web-ready documentation of XML and JSON schemas based on a metaschema. -parameters: none +## metapath -The output is a JSON Schema (v 7) that can be used to provide structural and lexical validation over JSON (and YAML) data instances, testing conformance to models defined by the metaschema. +Provides support for parsing and mapping Metapath, the metaschema path language. -## Generate converters +This directory is a dependency for logic in converter generation, which uses it to match JSON in conversion into XML, and schema generation, which uses it to implement path traversal in constraints definition and implementation. -### XML to JSON converter +## schema-gen -source: metaschema (main module) +Logic to provide schemas for validating XML or JSON according to definitions provided in a metaschema. -XSLT: `nist-metaschema-MAKE-XML-TO-JSON-CONVERTER.xsl` +Generators for XSD and JSON Schema v7 are provided. -result: XSLT (suffix `*.xsl`) +Additionally, a partial implementation of Metaschema constraints via a Schematron cast is offered, as a basis for future work. -An XML instance valid to a given metaschema-defined model can be converted by the XSLT produced by this XSLT (operating on the metaschema), into an information-identical JSON representation, losslessly, valid to the analogous JSON Schema. +## testing -### JSON to XML converter +Some testing artifacts. -source: metaschema (main module) +Also find testing within each subdirectory, appropriate to its functionalities. -XSLT: `nist-metaschema-MAKE-JSON-TO-XML-CONVERTER.xsl` +## util -result: XSLT (suffix `*.xsl`) +Miscellaneous utilities. Due for cleanup. -A JSON serialization (string) valid to a given metaschema-defined model can be converted by the XSLT produced by this XSLT (operating on the metaschema), into an information-identical XML representation, losslessly, valid to the analogous XML Schema (XSD). +## validate -## Generate Metatron / Metaschema-based constraints validation +Provides support for *extra-schema validation* of Metaschema instances against constraints implicit in Metaschema semantics. -source: metaschema (main module) +Note that validation provided here via Schematron applies *additionally* to regular structural schema validation of a metaschema, using its XSD. -XSLT: `nist-metaschema-MAKE-XML-METATRON.xsl` - -result: Schematron (suffix `*.sch`) - -tbd: Schematron that operates on JSON inputs (JSONatron) - -## Generate documentation - -For any metaschema a range of documentation artifacts are produced for consumption by Hugo (ingest into a static published documentation repository / web sites). - -Accordingly see these XProc pipelines for details: - -`make-metaschema-standalone-docs.xpl` a generic pipeline producing standalone documentation (by passing Hugo ingest files through a normalizer/stabilizer). - -Use with a debugging pipeline that binds the output ports for inspection. - -`write-hugo-metaschema-docs.xpl` producing the same set of docs, except writing them to the file system ready for Hugo. Note that this pipeline writes files to the system. - -Produced by both these pipelines (which should be work-alikes): - - XML and JSON-oriented model documents with cross-links - - Both instance- and model-oriented - - XML and JSON model maps / synopsis - - Indexes - -## Extras - -The XSLT `nist-metaschema-metaprocess.xsl` is a utility XSLT providing a unified interface for orchestrating the order and application of subordinate transformations, via configurations. - -### Metaschema schemas / `validate` folder - -Any metaschema, metaschema module, or composed metaschema, should be valid to the Metaschema XSD `../support/metaschema/schema/xml/metaschema.xsd` and to the `validate/metaschema-check.sch` Schematron. - -A composed metaschema is essentially what a metaschema will look like with all imports resolved (last appearing definition prevailing, imports read before main definitions); so a metaschema with no imports maps directly to its own composed expression. In composition, pointers are also written into the metaschema representation to provide useful information for downstream processing in resolving referential ambiguities (resulting from unintended or intended import clashes). - -The Schematron currently runs the composition step irrespectively. We should perhaps factor out Schematron checks that are dependent on Metaschema composition, from those that should apply to any metaschema (composed, standalone) or module. - -### Compose metaschema - -Schema composition is essentially import resolution, wherein overriding imports are resolved. Semantics of metaschema composition essentially follow Metaschema specifications for resolution of *metaschema modules* in importing. - -The result is a single normalized and annotated Metaschema module instance with all defaults explicit and disambiguated, or error reports. - -source: metaschema (main module) - -XSLT: `nist-metaschema-MAKE-JSON-MAP.xsl` - -result: XML conformant to Metaschema XSD/Schematron - -A composition step is provided internally by other processes, but it can also be run independently. - -### XProc - -As noted above, everything can also be done under XProc 1.0 (`*.xpl` files) for debugging. - -For a view of any XProc, try the [XProc Visualizer](https://pages.nist.gov/xslt-blender/xproc-visualizer/). - -Porting to XProc 3.0 and/or to other pipelining approaches is on the further horizon. +[end] diff --git a/src/common/nist-metaschema-metaprocess.xsl b/src/common/nist-metaschema-metaprocess.xsl index 7d7cdce4..a00d8461 100644 --- a/src/common/nist-metaschema-metaprocess.xsl +++ b/src/common/nist-metaschema-metaprocess.xsl @@ -18,7 +18,7 @@ - + - + - COMPOSING METASCHEMA { document-uri($source) } + COMPOSING METASCHEMA { base-uri($source) } @@ -48,7 +48,7 @@ - + @@ -68,7 +68,7 @@ - + diff --git a/src/converter-gen/mvn-jsonxml-converter-xsl.sh b/src/converter-gen/mvn-jsonxml-converter-xsl.sh old mode 100644 new mode 100755 diff --git a/src/converter-gen/mvn-xmljson-converter-xsl.sh b/src/converter-gen/mvn-xmljson-converter-xsl.sh old mode 100644 new mode 100755 diff --git a/src/document/mvn-schemadocs-debug-xpl.sh b/src/document/mvn-schemadocs-debug-xpl.sh old mode 100644 new mode 100755 diff --git a/src/document/mvn-schemadocs-html-xpl.sh b/src/document/mvn-schemadocs-html-xpl.sh old mode 100644 new mode 100755 diff --git a/src/schema-gen/mvn-xsd-schema-xsl.sh b/src/schema-gen/mvn-xsd-schema-xsl.sh old mode 100644 new mode 100755 diff --git a/src/testing/models_metaschema.xml b/src/testing/models_metaschema.xml index 7ed56698..e46580ba 100644 --- a/src/testing/models_metaschema.xml +++ b/src/testing/models_metaschema.xml @@ -25,24 +25,24 @@ ID - flag id + flag id. Simple Flag - some string + some string. Integer Flag - an integer + an integer.

Things to know about integers

EVERYTHING - assembly everything + assembly everything. EVERYTHING @@ -103,7 +103,7 @@ OVERLOADED - Overloaded field + Overloaded field. @@ -114,89 +114,89 @@ FIELD-1ONLY - field field-1only + field field-1only. - + FIELD-BASE64 - field field-base64 + field field-base64. FIELD-BOOLEAN - field field-boolean + field field-boolean. FIELD-SIMPLE-GROUPABLE - simple field, groupable + simple field, groupable. FIELD-FLAGGED-GROUPABLE - field field-groupable + field field-groupable. Date flag - A flag for a date + A flag for a date. Decimal flag - A flag with a decimal value + A flag with a decimal value. FIELD-WRAPPABLE - field field-wrappable + field field-wrappable. FIELD-BY-KEY - field field-by-key + field field-by-key. FIELD-NAMED-VALUE - field field-named-value + field field-named-value. CUSTOM-VALUE-KEY FIELD-DYNAMIC-VALUE-KEY - field field-dynamic-value-key + field field-dynamic-value-key. COLOR - flag color + flag color. MARKUP-LINE - field markup-line + field markup-line. WRAPPED-PROSE - field wrapped-prose + field wrapped-prose. LOOSE-PROSE - field loose-prose + field loose-prose. branch 1 for OVERLOAD1 - A discrete containter for objects with overloaded names. + A discrete containter for objects with overloaded names.. OVERLOAD @@ -207,7 +207,7 @@ branch 2 for OVERLOAD2 - A discrete containter for objects with overloaded names. + A discrete containter for objects with overloaded names.. OVERLOAD @@ -218,11 +218,11 @@ Overloaded name 1 - The first object with an overloaded name + The first object with an overloaded name. OVERLOAD An OVERLOAD/@a - A flag on an assembly called 'OVERLOAD' + A flag on an assembly called 'OVERLOAD'. @@ -234,11 +234,11 @@ Overloaded name 2 - The second object with an overloaded name + The second object with an overloaded name. OVERLOAD An OVERLOAD/@z - A flag on an assembly called 'OVERLOAD' + A flag on an assembly called 'OVERLOAD'. @@ -250,19 +250,19 @@ ASSEMBLY-EMPTY - assembly assembly-empty + assembly assembly-empty. ASSEMBLY-EMPTY-GROUPED - assembly assembly-empty-grouped + assembly assembly-empty-grouped. ASSEMBLY-1ONLY - assembly assembly-1only + assembly assembly-1only. ASSEMBLY-1ONLY @@ -272,7 +272,7 @@ ASSEMBLY-ALIASED 1 - first aliased assembly (has a name conflict) + first aliased assembly (has a name conflict). @@ -284,7 +284,7 @@ ASSEMBLY-ALIASED 2 - second aliased assembly (has a name conflict) + second aliased assembly (has a name conflict). @@ -293,7 +293,7 @@ ASSEMBLY-GROUPABLE - assembly assembly-groupable + assembly assembly-groupable. @@ -306,7 +306,7 @@ ASSEMBLY-WRAPPABLE - assembly assembly-wrappable + assembly assembly-wrappable. @@ -320,7 +320,7 @@ ASSEMBLY-BY-KEY - assembly assembly-by-key + assembly assembly-by-key. diff --git a/src/validate/METASCHEMA-SCHXSLT-VALIDATE.xpl b/src/validate/METASCHEMA-SCHXSLT-VALIDATE.xpl new file mode 100644 index 00000000..53ce2154 --- /dev/null +++ b/src/validate/METASCHEMA-SCHXSLT-VALIDATE.xpl @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/validate/METASCHEMA-VALIDATE.xpl b/src/validate/METASCHEMA-VALIDATE.xpl new file mode 100644 index 00000000..c0668bb2 --- /dev/null +++ b/src/validate/METASCHEMA-VALIDATE.xpl @@ -0,0 +1,52 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/validate/Makefile b/src/validate/Makefile new file mode 100644 index 00000000..665c7ee4 --- /dev/null +++ b/src/validate/Makefile @@ -0,0 +1,11 @@ +include ../testing/make_common.mk + +# XML Calabash prefers absolute paths +test_metaschema:=../testing/models_metaschema.xml + +.PHONY: test +test: smoke-test ## Run all tests + +.PHONY: smoke-test +smoke-test: ## Run all smoke-tests + ./mvn-composition-validate-xpl.sh "$(test_metaschema)" diff --git a/src/validate/metaschema-composition-check.sch b/src/validate/metaschema-composition-check.sch index 45a6333a..63a472af 100644 --- a/src/validate/metaschema-composition-check.sch +++ b/src/validate/metaschema-composition-check.sch @@ -3,8 +3,7 @@ xmlns:xsl="http://www.w3.org/1999/XSL/Transform" xmlns:m="http://csrc.nist.gov/ns/oscal/metaschema/1.0" xmlns:nm="http://csrc.nist.gov/ns/metaschema" - xmlns:sqf="http://www.schematron-quickfix.com/validator/process" - xmlns:oscal="http://csrc.nist.gov/ns/oscal/1.0"> + xmlns:sqf="http://www.schematron-quickfix.com/validator/process"> - - + + - @@ -61,12 +59,14 @@ Defs ID: Defs: '' --> - - - + + + + No appears named "" ... please check - Ambiguous reference to '' found in . The reference resolved to define- with the name '' in: . Is this due to a duplicated METASCHEMA/short-name in a module? + + Ambiguous reference to '' found in . The reference resolved to define- with the name '' in: . Is this due to a duplicated METASCHEMA/short-name in a module? @@ -86,26 +86,26 @@ Name clash among sibling elements or attributes with XML name ''. - - + + - Duplicate name found for '' in: . Is this due to a duplicated METASCHEMA/short-name in a module? + Duplicate name found for '' in: . Is this due to a duplicated METASCHEMA/short-name in a module? - + - Definition shadows another definition in this (composed) metaschema: see () + Definition shadows another definition in this (composed) metaschema: see () - + Unless @max-occurs is 1, a group-as name must be given within an instance or a local definition. @@ -116,11 +116,11 @@ - + - + @@ -165,8 +165,8 @@ - - + + Only one field may be marked as 'markup-multiline' (without xml wrapping) within a model. An 'unwrapped' field must have a max occurrence of 1 Only 'markup-multiline' fields may be unwrapped in XML. SEEING '' @@ -182,7 +182,10 @@ - JSON key indicates no flag on this - @flag-ref should be (one of) + + JSON key indicates no flag on this + + @@ -207,8 +210,6 @@ - - Formal name missing from named '' @@ -226,108 +227,5 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/src/validate/metaschema-validation-support.xsl b/src/validate/metaschema-validation-support.xsl index e818ed65..0bcfec9d 100644 --- a/src/validate/metaschema-validation-support.xsl +++ b/src/validate/metaschema-validation-support.xsl @@ -1,6 +1,7 @@ - + - + + + + + - - + + + ../compose/metaschema-collect.xsl ../compose/metaschema-build-refs.xsl ../compose/metaschema-trim-extra-modules.xsl @@ -37,10 +44,10 @@ - + - + @@ -51,8 +58,113 @@ - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/validate/mvn-composition-validate-xpl.sh b/src/validate/mvn-composition-validate-xpl.sh new file mode 100755 index 00000000..e5eee7a1 --- /dev/null +++ b/src/validate/mvn-composition-validate-xpl.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=../common/subcommand_common.bash +source "$SCRIPT_DIR/../common/subcommand_common.bash" + +usage() { + cat < + + + + + + + + + + + + + + + You are good - well-formed, no reported errors + + + + + + + + + : + + + + [ + + ] + + - + + - + + + + + + + + Q\{(.*?)\} + + + + + : + + + + + + + + + + + + + + diff --git a/support/schematron b/support/schematron deleted file mode 160000 index e16ecc49..00000000 --- a/support/schematron +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e16ecc490f9c6429f275ea268279787a71ff298f diff --git a/support/schxslt b/support/schxslt new file mode 160000 index 00000000..87cf981d --- /dev/null +++ b/support/schxslt @@ -0,0 +1 @@ +Subproject commit 87cf981dcefd80247d2203fbdf63f5f15f212e1d