#917 - Add test-harness module + change json schema test API to return list of errors

docs/PLUGINS.md

@@ -337,6 +337,6 @@ Add a test for the YAML file
 ```java
     @Test
     public void validSchemaShouldSucceed() throws Exception {
-        assertTrue(validateSchema(convertYamlFileToJson(this, "validJenkinsConfigurator.yml")));
+        assertThat(validateSchema(convertYamlFileToJson(this, "validSchemaConfig.yml")), empty());
     }
 ```

plugin/src/test/java/io/jenkins/plugins/casc/SchemaGenerationTest.java

@@ -8,9 +8,10 @@
 import static io.jenkins.plugins.casc.SchemaGeneration.retrieveDocStringFromAttribute;
 import static io.jenkins.plugins.casc.misc.Util.convertYamlFileToJson;
 import static io.jenkins.plugins.casc.misc.Util.validateSchema;
-import static junit.framework.TestCase.assertTrue;
+import static org.hamcrest.Matchers.contains;
+import static org.hamcrest.Matchers.empty;
 import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertThat;
 
 public class SchemaGenerationTest {
 
@@ -19,12 +20,13 @@ public class SchemaGenerationTest {
 
     @Test
     public void validSchemaShouldSucceed() throws Exception {
-        assertTrue(validateSchema(convertYamlFileToJson(this, "validSchemaConfig.yml")));
+        assertThat(validateSchema(convertYamlFileToJson(this, "validSchemaConfig.yml")), empty());
     }
 
     @Test
     public void invalidSchemaShouldNotSucceed() throws Exception {
-        assertFalse(validateSchema(convertYamlFileToJson(this,"invalidSchemaConfig.yml")));
+        assertThat(validateSchema(convertYamlFileToJson(this, "invalidSchemaConfig.yml")),
+            empty());
     }
 
     @Test

plugin/src/test/java/io/jenkins/plugins/casc/misc/Util.java

@@ -17,20 +17,24 @@
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
+import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import jenkins.model.GlobalConfigurationCategory;
 import jenkins.tools.ToolConfigurationCategory;
 import org.everit.json.schema.Schema;
+import org.everit.json.schema.ValidationException;
 import org.everit.json.schema.loader.SchemaLoader;
 import org.json.JSONObject;
 import org.json.JSONTokener;
 import org.jvnet.hudson.test.LoggerRule;
 
 import static io.jenkins.plugins.casc.ConfigurationAsCode.serializeYamlNode;
 import static io.jenkins.plugins.casc.SchemaGeneration.generateSchema;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.singletonList;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 
@@ -182,7 +186,7 @@ public static String toStringFromYamlFile(Object clazz, String resourcePath) thr
      */
     public static void assertNotInLog(LoggerRule logging, String unexpectedText) {
         assertFalse("The log should not contain '" + unexpectedText + "'",
-                logging.getMessages().stream().anyMatch(m -> m.contains(unexpectedText)));
+            logging.getMessages().stream().anyMatch(m -> m.contains(unexpectedText)));
     }
 
     /**
@@ -193,17 +197,17 @@ public static void assertNotInLog(LoggerRule logging, String unexpectedText) {
      */
     public static void assertLogContains(LoggerRule logging, String expectedText) {
         assertTrue("The log should contain '" + expectedText + "'",
-                logging.getMessages().stream().anyMatch(m -> m.contains(expectedText)));
+            logging.getMessages().stream().anyMatch(m -> m.contains(expectedText)));
     }
 
     /**
-     * Converts a given yamlString into a JsonString.
-     * Example Usage:
+     * <p>Converts a given yamlString into a JsonString.</p>
+     * <p>Example Usage:</p>
      * <pre>{@code
      * String jsonString = convertToJson(yourYamlString);}
      * </pre>
-     * @param yamlString
-     * @return JsonString
+     * @param yamlString the yaml to convert
+     * @return the json conversion of the yaml string.
      */
 
     public static String convertToJson(String yamlString) {
@@ -215,11 +219,12 @@ public static String convertToJson(String yamlString) {
 
     /**
      * Retrieves the JSON schema for the running jenkins instance.
-     * Example Usage:
-     *      * <pre>{@code
-     *      * Schema jsonSchema = returnSchema();}
-     *      * </pre>
-     * @return Schema the schema for the current jenkins instance
+     * <p>Example Usage:</p>
+     * <pre>{@code
+     *      Schema jsonSchema = returnSchema();}
+     *      </pre>
+     *
+     * @return the schema for the current jenkins instance
      */
     public static Schema returnSchema() throws Exception{
         JSONObject schemaObject = generateSchema();
@@ -229,34 +234,47 @@ public static Schema returnSchema() throws Exception{
     }
 
     /**
-     * Validates a given jsonObject against the schema generated for the current live jenkins instance
-     *      * Example Usage:
-     *      *      * <pre>{@code
-     *      *      * assertTrue(validateSchema(jsonSubject));}
-     *      *      * </pre>
-     * @param  jsonSubject The json Object that needs to be validated
-     * @return true if it's valid else returns false
+     * Validates a given jsonObject against the schema generated for the current live jenkins
+     * instance.
+     *
+     * <p>Example Usage:</p>
+     *  <pre>{@code
+     *   assertThat(validateSchema(convertYamlFileToJson(this, "invalidSchemaConfig.yml")),
+     *             contains("#/jenkins/numExecutors: expected type: Number, found: String"));
+     *  }</pre
+     *
+     *  <pre>{@code
+     *   assertThat(validateSchema(convertYamlFileToJson(this, "validConfig.yml")),
+     *             empty());
+     *  }</pre>
+     *
+     * @param jsonSubject the json object that needs to be validated
+     * @return a list of validation errors, empty list if no errors
      */
-    public static boolean validateSchema(JSONObject jsonSubject) {
+    public static List<String> validateSchema(JSONObject jsonSubject) {
         try {
             returnSchema().validate(jsonSubject);
+        } catch (ValidationException e) {
+            return e.getAllMessages();
         } catch (Exception ie) {
             LOGGER.log(Level.WARNING, failureMessage, ie);
-            return false;
+            return singletonList("Exception during test" + ie.getMessage());
         }
-        return true;
+        return emptyList();
     }
 
     /**
      * Converts a YAML file into a json object
-     * Example Usage:
-     *           * <pre>{@code
-     *           * JSONObject  jsonObject = convertYamlFileToJson("filename");}
-     *           * </pre>
+     * <p>Example Usage:</p>
+     * <pre>{@code
+     *  JSONObject jsonObject = convertYamlFileToJson(this, "filename");}
+     * </pre>
+     *
+     * @param clazz the class used for loading resources, normally you want to pass 'this'
      * @param yamlFileName the name of the yaml file that needs to be converted
      * @return JSONObject pertaining to that yaml file.
      */
-     public static JSONObject convertYamlFileToJson(Object clazz, String yamlFileName) throws Exception {
+    public static JSONObject convertYamlFileToJson(Object clazz, String yamlFileName) throws Exception {
         String yamlStringContents = toStringFromYamlFile(clazz, yamlFileName);
         return new JSONObject(new JSONTokener(convertToJson(yamlStringContents)));
     }