Merge pull request #815 from NASA-PDS/i681

Update precision check to be less than or equal to
NASA-PDS · Jan 23, 2024 · e71a398 · e71a398
2 parents e8c4912 + a71c74c
commit e71a398
Show file tree

Hide file tree

Showing 6 changed files with 168 additions and 49 deletions.
diff --git a/pom.xml b/pom.xml
@@ -1,44 +1,12 @@
 <?xml version="1.0" encoding="utf-8"?>
-<!--
-Copyright 2019–2021, California Institute of Technology ("Caltech").
-U.S. Government sponsorship acknowledged.
-
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
-• Redistributions of source code must retain the above copyright notice,
-  this list of conditions and the following disclaimer.
-• Redistributions must reproduce the above copyright notice, this list of
-  conditions and the following disclaimer in the documentation and/or other
-  materials provided with the distribution.
-• Neither the name of Caltech nor its operating division, the Jet Propulsion
-  Laboratory, nor the names of its contributors may be used to endorse or
-  promote products derived from this software without specific prior written
-  permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
--->
-
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
 
   <!-- Inherit release profile, reporting, SNAPSHOT repo, etc. from parent repo -->
   <parent>
     <groupId>gov.nasa</groupId>
     <artifactId>pds</artifactId>
-    <version>1.13.0</version>
+    <version>1.15.0</version>
   </parent>
 
   <groupId>gov.nasa.pds</groupId>
@@ -262,11 +230,11 @@ POSSIBILITY OF SUCH DAMAGE.
               
          Better ideas for how to do this welcome!
      -->
-    <!--dependency>
-      <groupId>org.slf4j</groupId>
-      <artifactId>slf4j-simple</artifactId>
-      <version>1.7.28</version>
-    </dependency-->
+<!--    <dependency>-->
+<!--      <groupId>org.slf4j</groupId>-->
+<!--      <artifactId>slf4j-simple</artifactId>-->
+<!--      <version>1.7.28</version>-->
+<!--    </dependency>-->
      <dependency>
       <groupId>org.slf4j</groupId>
       <artifactId>slf4j-nop</artifactId>
@@ -349,7 +317,7 @@ POSSIBILITY OF SUCH DAMAGE.
     <dependency>
       <groupId>gov.nasa.pds</groupId>
       <artifactId>pds4-jparser</artifactId>
-      <version>2.7.1</version>
+      <version>2.8.0-SNAPSHOT</version>
       <scope>compile</scope>
       <exclusions>
       <exclusion>

diff --git a/src/main/java/gov/nasa/pds/tools/validate/content/table/FieldValueValidator.java b/src/main/java/gov/nasa/pds/tools/validate/content/table/FieldValueValidator.java
@@ -360,8 +360,12 @@ public void validate(TableRecord record, FieldDescription[] fields, boolean chec
           if (checkFieldFormat) {
             // Due to CCB-214, the tool should validate against the
             // validation_format field for Character Tables.
-            if (record instanceof FixedTableRecord && !fields[i].getValidationFormat().isEmpty()) {
-              checkFormat(value, fields[i].getValidationFormat(), i + 1, record.getLocation());
+            if (record instanceof FixedTableRecord && (!fields[i].getValidationFormat().isEmpty() || !fields[i].getFieldFormat().isEmpty())) {
+              String format = fields[i].getValidationFormat();
+              if (format.isEmpty())
+                format = fields[i].getFieldFormat();
+
+              checkFormat(value, format, i + 1, record.getLocation());
             }
             if (record instanceof DelimitedTableRecord && !fields[i].getFieldFormat().isEmpty()) {
               checkFormat(value, fields[i].getFieldFormat(), i + 1, record.getLocation());
@@ -833,19 +837,22 @@ private void checkFormat(String value, String format, int fieldIndex,
         isValid = false;
       }
       if (precision != -1) {
+        // Per StdRef 4B.1.2, field_format defines the maximum precision, not the only precision allowed:
+        // For character tables, <field_format> is used to describe the maximum length and alignment of
+        // the data. <field_format> also gives an indication of the maximum precision of real numbers, but
+        // does not require all values to have this precision.
         if (specifier.matches("[feE]")) {
           String[] tokens = value.trim().split("[eE]", 2);
-          int length = 0;
+          int actual_precision = 0;
           if (tokens[0].indexOf(".") != -1) {
-            length = tokens[0].substring(tokens[0].indexOf(".") + 1).length();
+            actual_precision = tokens[0].substring(tokens[0].indexOf(".") + 1).length();
           }
-          if (length != precision) {
+          if (actual_precision > precision) {
             isValid = false;
             addTableProblem(ExceptionType.ERROR, ProblemType.FIELD_VALUE_FORMAT_PRECISION_MISMATCH,
                 "The number of digits to the right of the decimal point " + "in the value '"
-                    + value.trim() + "' does not equal the "
-                    + "precision set in the defined field format " + "(expected " + precision
-                    + ", got " + length + ").",
+                    + value.trim() + "' must be <= the precision set in the defined field format '"
+                    + format+ "' (Expected: <=" + precision + ", Actual: " + actual_precision + ").",
                     recordLocation, fieldIndex);
           }
         }

diff --git a/src/test/resources/features/developer.feature b/src/test/resources/features/developer.feature
@@ -28,7 +28,8 @@ Scenario Outline: Execute validate command for tests below.
 |"NASA-PDS/validate#690 Success new constant expression" | "github690" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources"   | "target/test" | "-r {reportDir}/report_github690.json -s json --skip-context-validation  -t {resourceDir}/github690/rs_20160518_014000_udsc64_l3_e_v10.xml" | "report_github690.json" |
 
 # Validate#681
-|"NASA-PDS/validate#681 Success and Failure of ASCII table with space in number" | "github681" | 1 | "1 errors expected" | "FIELD_VALUE_FORMAT_PRECISION_MISMATCH" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github681.json -s json --skip-context-validation  -t {resourceDir}/github681/ff_char.xml {resourceDir}/github681/ff_del.xml" | "report_github681.json" |
+|"NASA-PDS/validate#681 Success ASCII table with extra whitespace, valid precision" | "github681" | 0 | "0 errors expected" | "FIELD_VALUE_FORMAT_PRECISION_MISMATCH" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github681_1.json -s json  -t {resourceDir}/github681/ff_char.xml {resourceDir}/github681/ff_del.xml" | "report_github681_1.json" |
+|"NASA-PDS/validate#681 Failure of ASCII table invalid precision" | "github681" | 2 | "2 errors expected" | "FIELD_VALUE_FORMAT_PRECISION_MISMATCH" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github681_2.json -s json -t {resourceDir}/github681/ff_char_fail.xml {resourceDir}/github681/ff_del_fail.xml" | "report_github681_2.json" |
 
 # Validate#680
 |"NASA-PDS/validate#680 Success char table correct length" | "github680" | 0 | "0 errors expected" | "totalErrors" | "src/test/resources" | "target/test" | "-r {reportDir}/report_github680.1.json -s json --skip-context-validation  -t {resourceDir}/github680/ORB12_EUR_EPHIO_reclen96.xml" | "report_github680.1.json" |

diff --git a/src/test/resources/github681/ff_char_fail.xml b/src/test/resources/github681/ff_char_fail.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1K00.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<Product_Observational xmlns="http://pds.nasa.gov/pds4/pds/v1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pds.nasa.gov/pds4/pds/v1 https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1K00.xsd">
+  <Identification_Area>
+    <logical_identifier>urn:nasa:pds:bundle:collection:ff_char_test</logical_identifier>
+    <version_id>1.0</version_id>
+    <title>Field Format Character Test</title>
+    <information_model_version>1.20.0.0</information_model_version>
+    <product_class>Product_Observational</product_class>
+  </Identification_Area>
+  <Observation_Area>
+    <Time_Coordinates>
+      <start_date_time>2023Z</start_date_time>
+      <stop_date_time>2023Z</stop_date_time>
+    </Time_Coordinates>
+    <Investigation_Area>
+      <name>None</name>
+      <type>Other Investigation</type>
+      <Internal_Reference>
+        <lid_reference>urn:nasa:pds:context:investigation:individual.none</lid_reference>
+        <reference_type>data_to_investigation</reference_type>
+      </Internal_Reference>
+    </Investigation_Area>
+    <Observing_System>
+      <Observing_System_Component>
+        <name>telescope</name>
+        <type>Instrument</type>
+      </Observing_System_Component>
+    </Observing_System>
+    <Target_Identification>
+      <name>target</name>
+      <type>Comet</type>
+    </Target_Identification>
+  </Observation_Area>
+  <File_Area_Observational>
+    <File>
+      <file_name>ff_test_fail.csv</file_name>
+    </File>
+    <Table_Character>
+      <offset unit="byte">0</offset>
+      <records>3</records>
+      <record_delimiter>Carriage-Return Line-Feed</record_delimiter>
+      <Record_Character>
+        <fields>3</fields>
+        <groups>0</groups>
+        <record_length unit="byte">22</record_length>
+        <Field_Character>
+          <name>field 1</name>
+          <field_number>1</field_number>
+          <field_location unit="byte">1</field_location>
+          <data_type>ASCII_Real</data_type>
+          <field_length unit="byte">6</field_length>
+          <field_format>%6.2f</field_format>
+        </Field_Character>
+        <Field_Character>
+          <name>field 2</name>
+          <field_number>2</field_number>
+          <field_location unit="byte">8</field_location>
+          <data_type>ASCII_Real</data_type>
+          <field_length unit="byte">4</field_length>
+          <field_format>%4.2f</field_format>
+        </Field_Character>
+        <Field_Character>
+          <name>field 3</name>
+          <field_number>3</field_number>
+          <field_location unit="byte">13</field_location>
+          <data_type>ASCII_Real</data_type>
+          <field_length unit="byte">8</field_length>
+          <field_format>%8.3e</field_format>
+        </Field_Character>
+      </Record_Character>
+    </Table_Character>
+  </File_Area_Observational>
+</Product_Observational>
diff --git a/src/test/resources/github681/ff_del_fail.xml b/src/test/resources/github681/ff_del_fail.xml
@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1K00.sch" schematypens="http://purl.oclc.org/dsdl/schematron"?>
+<Product_Observational xmlns="http://pds.nasa.gov/pds4/pds/v1" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://pds.nasa.gov/pds4/pds/v1 https://pds.nasa.gov/pds4/pds/v1/PDS4_PDS_1K00.xsd">
+  <Identification_Area>
+    <logical_identifier>urn:nasa:pds:bundle:collection:ff_del_test</logical_identifier>
+    <version_id>1.0</version_id>
+    <title>Field Format Delimited Test</title>
+    <information_model_version>1.20.0.0</information_model_version>
+    <product_class>Product_Observational</product_class>
+  </Identification_Area>
+  <Observation_Area>
+    <Time_Coordinates>
+      <start_date_time>2023Z</start_date_time>
+      <stop_date_time>2023Z</stop_date_time>
+    </Time_Coordinates>
+    <Investigation_Area>
+      <name>None</name>
+      <type>Other Investigation</type>
+      <Internal_Reference>
+        <lid_reference>urn:nasa:pds:context:investigation:individual.none</lid_reference>
+        <reference_type>data_to_investigation</reference_type>
+      </Internal_Reference>
+    </Investigation_Area>
+    <Observing_System>
+      <Observing_System_Component>
+        <name>telescope</name>
+        <type>Instrument</type>
+      </Observing_System_Component>
+    </Observing_System>
+    <Target_Identification>
+      <name>target</name>
+      <type>Comet</type>
+    </Target_Identification>
+  </Observation_Area>
+  <File_Area_Observational>
+    <File>
+      <file_name>ff_test_fail.csv</file_name>
+    </File>
+    <Table_Delimited>
+      <offset unit="byte">0</offset>
+      <parsing_standard_id>PDS DSV 1</parsing_standard_id>
+      <records>3</records>
+      <record_delimiter>Carriage-Return Line-Feed</record_delimiter>
+      <field_delimiter>Comma</field_delimiter>
+      <Record_Delimited>
+        <fields>3</fields>
+        <groups>0</groups>
+        <Field_Delimited>
+          <name>field 1</name>
+          <field_number>1</field_number>
+          <data_type>ASCII_Real</data_type>
+          <field_format>%6.2f</field_format>
+        </Field_Delimited>
+        <Field_Delimited>
+          <name>field 2</name>
+          <field_number>2</field_number>
+          <data_type>ASCII_Real</data_type>
+          <field_format>%4.2f</field_format>
+        </Field_Delimited>
+        <Field_Delimited>
+          <name>field 3</name>
+          <field_number>3</field_number>
+          <data_type>ASCII_Real</data_type>
+          <field_format>%8.3e</field_format>
+        </Field_Delimited>
+      </Record_Delimited>
+    </Table_Delimited>
+  </File_Area_Observational>
+</Product_Observational>
diff --git a/.../resources/github681/ff_test_no_space.csv → ...test/resources/github681/ff_test_fail.csv b/.../resources/github681/ff_test_no_space.csv → ...test/resources/github681/ff_test_fail.csv
@@ -1,3 +1,3 @@
 124.64,8.29,8.623e04
-154.64,2.2,8.653e04
+154.64,.222,8.653e04
 194.64,8.93,8.673e04