Addressing datatype validation issues: whitespace collapsing; non-emp…

…ty values; ncname-workalike in JSON Schema - see usnistgov/OSCAL#911 usnistgov/OSCAL#805 also usnistgov#33 usnistgov#67 usnistgov#68
wendellpiez · May 21, 2021 · ea371c6 · ea371c6
1 parent 16a32f8
commit ea371c6
Show file tree

Hide file tree

Showing 4 changed files with 84 additions and 9 deletions.
diff --git a/toolchains/xslt-M4/schema-gen/make-json-schema-metamap.xsl b/toolchains/xslt-M4/schema-gen/make-json-schema-metamap.xsl
@@ -630,6 +630,8 @@
         <dummy/>
     </xsl:variable>-->
 
+
+    <!-- Must be kept aligned with oscal-datatypes.xsd... -->
     <xsl:variable name="datatypes" expand-text="false">
         <map key="decimal">
             <string key="type">number</string>
@@ -696,7 +698,40 @@
             <!-- A sequence of 8-4-4-4-12 hex digits, with extra constraints in the 13th and 17-18th places for version 4-->
             <string key="pattern">^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-4[0-9A-Fa-f]{3}-[89ABab][0-9A-Fa-f]{3}-[0-9A-Fa-f]{12}$</string>
         </map>
-        <!-- Possibly add support for XSD types ID, IDREF, IDREFS, NCName, NMTOKENS ???        -->
+        <map key="token">
+            <!-- 
+Emulating XSD [\i-[:]][\c-[:]]*
+
+From https://www.w3.org/TR/xml11/#NT-NameChar
+
+PRODUCTION 4 NameStartChar	   ::=
+   	":" | [A-Z] | "_" | [a-z] | [#xC0-#xD6] | [#xD8-#xF6] | [#xF8-#x2FF] | [#x370-#x37D] | [#x37F-#x1FFF] | [#x200C-#x200D] | [#x2070-#x218F] | [#x2C00-#x2FEF] | [#x3001-#xD7FF] | [#xF900-#xFDCF] | [#xFDF0-#xFFFD] | [#x10000-#xEFFFF]
+
+excluding [#x10000-#xEFFFF] since we can't match them in Javascript regex without extra somersaults
+
+-> Javascript regex NameStartChar (no colon) [_A-Za-z\uC0-\uD6\uD8-\uF6\uF8-\u2FF\u370-\u37D\u37F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]
+
+PRODUCTION 4a NameChar	   ::=
+   	NameStartChar | "-" | "." | [0-9] | #xB7 | [#x0300-#x036F] | [#x203F-#x2040]
+
+-> Javascript regex NameChar (no colon)
+[_\-\.\u00B7\u0300-\u036F\u203F-\u2040 + NameStartChar ]
+
+        -->
+            <string key="type">string</string>
+            <xsl:sequence expand-text="true">
+                <!--<xsl:variable name="test">_A-Z123\u00C0-\u00D6</xsl:variable>-->
+                <xsl:variable name="initial-name-chars" as="xs:string">_A-Za-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD<!--\u10000-\uEFFFF--></xsl:variable>
+                <xsl:variable as="xs:string" name="name-chars">{ $initial-name-chars }\-\.0-9\u00B7\u0300-\u036F\u203F-\u2040</xsl:variable>
+                <!--<string key="pattern">^[{ $test }]+$</string>-->
+                <string key="pattern">^[{ $initial-name-chars }][{ $name-chars }]*$</string>
+            </xsl:sequence>
+        </map>
+        <map key="string">
+            <string key="type">string</string>
+            <string key="pattern">^\S(.*\S)?$</string>
+            <!---->
+        </map>
     </xsl:variable>
 
 </xsl:stylesheet>
diff --git a/toolchains/xslt-M4/schema-gen/make-metaschema-xsd.xsl b/toolchains/xslt-M4/schema-gen/make-metaschema-xsd.xsl
@@ -444,7 +444,6 @@
             <xsl:attribute name="{ $assign-to-attribute }" expand-text="true"
                 select="concat($declaration-prefix, ':', $datatype)"/>
         </xsl:if>
-
     </xsl:template>
 
     <!-- When allow-other=yes, we union the enumeration with the declared datatype -->        

diff --git a/toolchains/xslt-M4/schema-gen/oscal-datatypes.xsd b/toolchains/xslt-M4/schema-gen/oscal-datatypes.xsd
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <xs:schema
-  xmlns="http://standards.iso.org/iso/19770/-2/2015/schema.xsd"
   xmlns:xs="http://www.w3.org/2001/XMLSchema"
   xmlns:m="http://csrc.nist.gov/ns/oscal/metaschema/1.0"
 
@@ -14,23 +13,39 @@
   <xs:appinfo>
     <xs:simpleType name="boolean"/>
     <xs:simpleType name="base64Binary"/>
-    <xs:simpleType name="string"/>
+    <!--<xs:simpleType name="string"/>-->
     <xs:simpleType name="NCName"/>
-    <xs:simpleType name="NMTOKENS"/>
+    <!--<xs:simpleType name="NMTOKENS"/>-->
     <xs:simpleType name="decimal"/>
     <!-- Not supporting float or double -->
     <!--<xs:simpleType name="float"/>
         <xs:simpleType name="double"/>-->
     <xs:simpleType name="integer"/>
     <xs:simpleType name="nonNegativeInteger"/>
     <xs:simpleType name="positiveInteger"/>
-    <xs:simpleType name="ID"/>
-    <xs:simpleType name="IDREF"/>
-    <xs:simpleType name="IDREFS"/>
+    <!--<xs:simpleType name="ID"/>-->
+    <!--<xs:simpleType name="IDREF"/>-->
+    <!--<xs:simpleType name="IDREFS"/>-->
     <xs:simpleType name="date"/>
     <xs:simpleType name="dateTime"/>
   </xs:appinfo>
 </xs:annotation>
+
+  <!-- The OSCAL 'string' datatype restricts the XSD type by prohibiting leading and trailing whitespace, and something
+     (not only whitespace) is required. -->
+  <xs:simpleType name="string">
+    <xs:annotation>
+      <xs:documentation>A string, but not empty and not whitespace-only (whitespace is U+9, U+10, U+32 or [ \n\t]+ )</xs:documentation>
+    </xs:annotation>
+    <xs:restriction base="xs:string">
+      <xs:whiteSpace value="preserve"/>
+      <xs:pattern value="\S(.*\S)?">
+        <xs:annotation>
+          <xs:documentation>A trimmed string, at least one character with no leading or trailing whitespace.</xs:documentation>
+        </xs:annotation>
+      </xs:pattern>
+    </xs:restriction>
+  </xs:simpleType>
 
 
   <xs:simpleType name="ip-v4-address">
@@ -137,5 +152,25 @@
     </xs:restriction>
   </xs:simpleType>
 
+  <!-- This could be renamed 'ncname-token' if we wish to loosen 'token', as this is pretty strict. -->
+  <xs:simpleType name="token">
+    <xs:annotation>
+      <!--<xs:documentation>Matching XSD NCName, except whitespace is not collapsed.</xs:documentation>-->
+      <xs:documentation>A string token following the rules of XML "no colon" names, with no whitespace. (XML names are single alphabetic characters
+      followed by alphanumeric characters, periods, underscores or dashes.)</xs:documentation>
+    </xs:annotation>
+    <xs:restriction base="xs:string">
+      <xs:whiteSpace value="preserve"/>
+      <xs:pattern value="[\i-[:]][\c-[:]]*">
+        <xs:annotation>
+          <!--<xs:documentation>An XML initial character (but not colon), followed by any XML name character (but not colon).</xs:documentation>-->
+          <xs:documentation>A single token may not contain whitespace.</xs:documentation>
+        </xs:annotation>
+      </xs:pattern>
+    </xs:restriction>
+  </xs:simpleType>
+
+
+
 </xs:schema>
 
diff --git a/toolchains/xslt-M4/validate/metaschema.xsd b/toolchains/xslt-M4/validate/metaschema.xsd
@@ -862,7 +862,11 @@
   <xs:simpleType name="simple-datatypes">
     <xs:restriction base="xs:string">
       <xs:enumeration value="boolean"/>
-      <xs:enumeration value="string"/>
+      <xs:enumeration value="string">
+        <xs:annotation>
+          <xs:documentation>A string with no leading or trailing whitespace.</xs:documentation>
+        </xs:annotation>
+      </xs:enumeration>
       <xs:enumeration value="decimal"/>
       <!-- Not supporting float or double -->
       <!--<xs:enumeration value="float" />
@@ -882,9 +886,11 @@
       <xs:enumeration value="hostname"/>
       <xs:enumeration value="ip-v4-address"/>
       <xs:enumeration value="ip-v6-address"/>
+      <xs:enumeration value="token"/>
       <xs:enumeration value="uri"/>
       <xs:enumeration value="uri-reference"/>
       <xs:enumeration value="uuid"/>
+
     </xs:restriction>
   </xs:simpleType>