Workflow url in run mode

CHANGELOG.md

@@ -2,10 +2,12 @@
 
 ## 35 Release Notes
 
-### Submit workflow API
+### Submit workflow using URL
 
-Cromwell now allows for a user to submit the URL pointing to workflow file to run a workflow using `workflowUrl` parameter. Currently, this is only supported in `Server` mode.
-More details on how to use it can be found [here](http://cromwell.readthedocs.io/en/develop/api/RESTAPI/).
+Cromwell now allows for a user to submit the URL pointing to workflow file to run a workflow.
+More details on how to use it in: 
+- `Server` mode can be found [here](https://cromwell.readthedocs.io/en/develop/api/RESTAPI/).
+- `Run` mode can be found [here](https://cromwell.readthedocs.io/en/develop/CommandLine/#run).
 
 ### Languages
 

centaur/src/main/resources/standardTestCases/invalid_workflow_url_length.test

@@ -0,0 +1,14 @@
+name: invalid_workflow_url_length
+testFormat: submitfailure
+
+files {
+  workflowUrl: "https://this_url_has_more_than_2000_characters/why_would_someone_have_such_long_urls_one_would_ask/beats_me/now_starts_lorem_ipsum/At_vero_eos_et_accusamus_et_iusto_odio_dignissimos_ducimus_qui_blanditiis_praesentium_voluptatum_deleniti_atque_corrupti_quos_dolores_et_quas_molestias_excepturi_sint_occaecati_cupiditate_non_provident,_similique_sunt_in_culpa_qui_officia_deserunt_mollitia_animi,_id_est_laborum_et_dolorum_fuga._Et_harum_quidem_rerum_facilis_est_et_expedita_distinctio._Nam_libero_tempore,_cum_soluta_nobis_est_eligendi_optio_cumque_nihil_impedit_quo_minus_id_quod_maxime_placeat_facere_possimus,_omnis_voluptas_assumenda_est,_omnis_dolor_repellendus._Temporibus_autem_quibusdam_et_aut_officiis_debitis_aut_rerum_necessitatibus_saepe_eveniet_ut_et_voluptates_repudiandae_sint_et_molestiae_non_recusandae._Itaque_earum_rerum_hic_tenetur_a_sapiente_delectus,_ut_aut_reiciendis_voluptatibus_maiores_alias_consequatur_aut_perferendis_doloribus_asperiores_repellat/Sed_ut_perspiciatis_unde_omnis_iste_natus_error_sit_voluptatem_accusantium_doloremque_laudantium,_totam_rem_aperiam,_eaque_ipsa_quae_ab_illo_inventore_veritatis_et_quasi_architecto_beatae_vitae_dicta_sunt_explicabo._Nemo_enim_ipsam_voluptatem_quia_voluptas_sit_aspernatur_aut_odit_aut_fugit,_sed_quia_consequuntur_magni_dolores_eos_qui_ratione_voluptatem_sequi_nesciunt._Neque_porro_quisquam_est,_qui_dolorem_ipsum_quia_dolor_sit_amet,_consectetur,_adipisci_velit,_sed_quia_non_numquam_eius_modi_tempora_incidunt_ut_labore_et_dolore_magnam_aliquam_quaerat_voluptatem._Ut_enim_ad_minima_veniam,_quis_nostrum_exercitationem_ullam_corporis_suscipit_laboriosam,_nisi_ut_aliquid_ex_ea_commodi_consequatur?_Quis_autem_vel_eum_iure_reprehenderit_qui_in_ea_voluptate_velit_esse_quam_nihil_molestiae_consequatur,_vel_illum_qui_dolorem_eum_fugiat_quo_voluptas_nulla_pariatur?/Lorem_ipsum_dolor_sit_amet,_consectetur_adipiscing_elit,_sed_do_eiusmod_tempor_incididunt_ut_labore_et_dolore_magna_aliqua._Ut_enim_ad_minim_veniam,_quis_nostrud_exercitation_ullamco_laboris_nisi_ut_aliquip_ex_ea_commodo_consequat._Duis_aute_irure_dolor_in_reprehenderit_in_voluptate_velit_esse_cillum_dolore_eu_fugiat_nulla_pariatur._Excepteur_sint_occaecat_cupidatat_non_proident,_sunt_in_culpa_qui_officia_deserunt_mollit_anim_id_est_laborum/hello.wdl/hello.wdl"
+}
+
+submit {
+  statusCode: 400
+  message: """{
+  "status": "fail",
+  "message": "Error(s): Invalid workflow url: url has length 2305, longer than the maximum allowed 2000 characters"
+}"""
+}

database/migration/src/main/resources/changelog.xml

@@ -74,6 +74,7 @@
     <include file="changesets/workflow_store_workflow_root_column.xml" relativeToChangelogFile="true" />
     <include file="changesets/workflow_store_horizontal_db.xml" relativeToChangelogFile="true" />
     <include file="changesets/add_workflow_url_in_workflow_store_entry.xml" relativeToChangelogFile="true" />
+    <include file="changesets/change_max_size_workflow_url.xml" relativeToChangelogFile="true" />
 </databaseChangeLog>
 <!--
 

database/migration/src/main/resources/changesets/change_max_size_workflow_url.xml

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<databaseChangeLog xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+                   xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+                   xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-3.3.xsd">
+
+    <changeSet author="sshah" id="change_max_size_for_workflow_url">
+        <modifyDataType tableName="WORKFLOW_STORE_ENTRY" columnName="WORKFLOW_URL" newDataType="VARCHAR(2000)"/>
+    </changeSet>
+
+</databaseChangeLog>

database/sql/src/main/scala/cromwell/database/slick/tables/WorkflowStoreEntryComponent.scala

@@ -34,7 +34,7 @@ trait WorkflowStoreEntryComponent {
 
     def workflowDefinition = column[Option[Clob]]("WORKFLOW_DEFINITION")
 
-    def workflowUrl = column[Option[String]]("WORKFLOW_URL", O.Length(255))
+    def workflowUrl = column[Option[String]]("WORKFLOW_URL", O.Length(2000))
 
     def workflowInputs = column[Option[Clob]]("WORKFLOW_INPUTS")
 

docs/CommandLine.md

@@ -15,7 +15,8 @@ Command: server
 Starts a web server on port 8000.  See the web server documentation for more details about the API endpoints.
 Command: run [options] workflow-source
 Run the workflow and print out the outputs in JSON format.
-  workflow-source          Workflow source file.
+  workflow-source          Workflow source file or workflow url .
+  --workflow-root <value>  Workflow root
   -i, --inputs <value>     Workflow inputs file.
   -o, --options <value>    Workflow options file.
   -t, --type <value>       Workflow type.
@@ -40,7 +41,7 @@ The Cromwell jar file can be built as described in [Building](Building).
 `run` mode executes a single workflow in Cromwell and then exits.
 
 * **`workflow-source`**  
-The single required argument for the workflow source file.
+The single required argument. It can either be a workflow source file or a url pointing to the workflow.
 
 * **`--inputs`**  
 An optional file of workflow inputs.  Although optional, it is a best practice to use an inputs file to satisfy workflow

engine/src/main/scala/cromwell/webservice/PartialWorkflowSources.scala

@@ -3,7 +3,6 @@ package cromwell.webservice
 import java.net.URL
 
 import _root_.io.circe.yaml
-import akka.http.scaladsl.model.IllegalUriException
 import akka.util.ByteString
 import cats.data.NonEmptyList
 import cats.data.Validated._
@@ -59,6 +58,8 @@ object PartialWorkflowSources {
 
   val allPrefixes = List(WorkflowInputsAuxPrefix)
 
+  val MaxWorkflowUrlLength = 2000
+
   def fromSubmitRoute(formData: Map[String, ByteString],
                       allowNoInputs: Boolean): Try[Seq[WorkflowSourceFilesCollection]] = {
     import cats.instances.list._
@@ -206,16 +207,12 @@ object PartialWorkflowSources {
       }
     }
 
-    def validateWorkflowUrl(workflowUrl: Option[String]): ErrorOr[Option[WorkflowUrl]] = {
-      workflowUrl.traverse(convertStringToUrl)
-    }
-
     partialSources match {
       case Valid(partialSource) =>
         (validateInputs(partialSource),
           validateOptions(partialSource.workflowOptions),
           validateLabels(partialSource.customLabels.getOrElse("{}")),
-          validateWorkflowUrl(partialSource.workflowUrl)) mapN {
+          partialSource.workflowUrl.traverse(validateWorkflowUrl)) mapN {
           case (wfInputs, wfOptions, workflowLabels, wfUrl) =>
             wfInputs.map(inputsJson => WorkflowSourceFilesCollection(
               workflowSource = partialSource.workflowSource,
@@ -239,11 +236,17 @@ object PartialWorkflowSources {
     JsObject(convertToMap reduce (_ ++ _))
   }
 
-  def convertStringToUrl(workflowUrl: String): ErrorOr[WorkflowUrl] = {
-    Try(new URL(workflowUrl)) match {
-      case Success(_) => workflowUrl.validNel
-      case Failure(e: IllegalUriException) => s"Invalid workflow url: ${e.getMessage}".invalidNel
-      case Failure(e) => s"Error while validating workflow url: ${e.getMessage}".invalidNel
+  def validateWorkflowUrl(workflowUrl: String): ErrorOr[WorkflowUrl] = {
+    def convertStringToUrl(workflowUrl: String): ErrorOr[WorkflowUrl] = {
+      Try(new URL(workflowUrl)) match {
+        case Success(_) => workflowUrl.validNel
+        case Failure(e) => s"Error while validating workflow url: ${e.getMessage}".invalidNel
+      }
+    }
+
+    workflowUrl.length match {
+      case l if l > MaxWorkflowUrlLength => s"Invalid workflow url: url has length $l, longer than the maximum allowed $MaxWorkflowUrlLength characters".invalidNel
+      case _ => convertStringToUrl(workflowUrl)
     }
   }
 

server/src/main/scala/cromwell/CommandLineArguments.scala

@@ -1,6 +1,7 @@
 package cromwell
 
 import java.net.URL
+import java.nio.file.InvalidPathException
 
 import better.files.File
 import cats.syntax.apply._
@@ -16,7 +17,7 @@ import cromwell.webservice.PartialWorkflowSources
 import cwl.preprocessor.CwlPreProcessor
 import org.slf4j.Logger
 
-import scala.util.{Success, Try}
+import scala.util.{Failure, Success, Try}
 
 object CommandLineArguments {
   val DefaultCromwellHost = new URL("http://localhost:8000")
@@ -27,11 +28,12 @@ object CommandLineArguments {
                               workflowOptions: String,
                               workflowLabels: String,
                               dependencies: Option[File])
+
+  case class WorkflowSourceOrUrl(source: Option[String], url: Option[String])
 }
 
 case class CommandLineArguments(command: Option[Command] = None,
-                                workflowSource: Option[Path] = None,
-                                workflowUrl: Option[String] = None,
+                                workflowSource: Option[String] = None,
                                 workflowRoot: Option[String] = None,
                                 workflowInputs: Option[Path] = None,
                                 workflowOptions: Option[Path] = None,
@@ -60,26 +62,33 @@ case class CommandLineArguments(command: Option[Command] = None,
   }
 
   def validateSubmission(logger: Logger): ErrorOr[ValidSubmission] = {
-    val workflowPath = File(workflowSource.get.pathAsString)
+    def preProcessCwlWorkflowSource(workflowSourcePath: Path): ErrorOr[String] = {
+      val workflowPath = File(workflowSourcePath.pathAsString)
 
-    val workflowAndDependencies: ErrorOr[(String, Option[File], Option[String])] = if (isCwl) {
       logger.info("Pre Processing Workflow...")
       lazy val preProcessedCwl = cwlPreProcessor.preProcessCwlFileToString(workflowPath, None)
 
-      imports match {
-        case Some(explicitImports) => readOptionContent("Workflow source", workflowSource).map((_, Option(File(explicitImports.pathAsString)), workflowRoot))
-        case None => Try(preProcessedCwl.map((_, None, None)).value.unsafeRunSync())
-          .toChecked
-          .flatMap(identity)
-          .toValidated
+      Try(preProcessedCwl.value.unsafeRunSync())
+        .toChecked
+        .flatMap(identity)
+        .toValidated
+    }
+
+    def getWorkflowSourceFromPath(workflowPath: Path): ErrorOr[WorkflowSourceOrUrl] = {
+      (isCwl, imports) match {
+        case (true, None) => preProcessCwlWorkflowSource(workflowPath).map(src => WorkflowSourceOrUrl(Option(src), None))
+        case (true, Some(_)) | (false, _) => WorkflowSourceOrUrl(None, Option(workflowPath.pathAsString)).validNel
       }
-    } else readOptionContent("Workflow source", workflowSource).map((_, imports.map(p => File(p.pathAsString)), workflowRoot))
+    }
 
-    val workflowSourceFinal: ErrorOr[String] = (workflowSource, workflowUrl) match {
-      case (Some(path), None) => readContent("Workflow source", path)
-      case (None, Some(url)) =>  PartialWorkflowSources.convertStringToUrl(url)
-      case (Some(_), Some(_)) => "Both Workflow source and Workflow url can't be supplied".invalidNel
-      case (None, None) => "Workflow source and Workflow url needs to be supplied".invalidNel
+    val workflowSourceAndUrl: ErrorOr[WorkflowSourceOrUrl] = DefaultPathBuilder.build(workflowSource.get) match {
+      case Success(workflowPath) => {
+        if (!workflowPath.exists) s"Workflow source path does not exist: $workflowPath".invalidNel
+        else if(!workflowPath.isReadable) s"Workflow source path is not readable: $workflowPath".invalidNel
+        else getWorkflowSourceFromPath(workflowPath)
+      }
+      case Failure(e: InvalidPathException) => s"Invalid file path. Error: ${e.getMessage}".invalidNel
+      case Failure(_) => PartialWorkflowSources.validateWorkflowUrl(workflowSource.get).map(validUrl => WorkflowSourceOrUrl(None, Option(validUrl)))
     }
 
     val inputsJson: ErrorOr[String] = if (isCwl) {
@@ -90,9 +99,11 @@ case class CommandLineArguments(command: Option[Command] = None,
     val optionsJson = readOptionContent("Workflow options", workflowOptions)
     val labelsJson = readOptionContent("Workflow labels", workflowLabels)
 
-    (workflowAndDependencies, inputsJson, optionsJson, labelsJson, workflowSourceFinal) mapN {
-      case ((w, z, r), i, o, l, _) =>
-        ValidSubmission(Option(w), workflowUrl, r, i, o, l, z)
+    val workflowImports: Option[File] = imports.map(p => File(p.pathAsString))
+
+    (workflowSourceAndUrl, inputsJson, optionsJson, labelsJson) mapN {
+      case (srcOrUrl, i, o, l) =>
+        ValidSubmission(srcOrUrl.source, srcOrUrl.url, workflowRoot, i, o, l, workflowImports)
     }
   }
 

server/src/main/scala/cromwell/CommandLineParser.scala

@@ -20,9 +20,8 @@ object CommandLineParser {
 //  Starts a web server on port 8000.  See the web server documentation for more details about the API endpoints.
 //  Command: run [options] workflow-source
 //  Run the workflow and print out the outputs in JSON format.
-//  workflow-source          Workflow source file.
-//  -u, --workflow-url <value>
-//                           Workflow source url.
+//  workflow-source          Workflow source file or workflow url.
+//  --workflow-root <value>  Workflow root
 //  -i, --inputs <value>     Workflow inputs file.
 //  -o, --options <value>    Workflow options file.
 //  -t, --type <value>       Workflow type.
@@ -36,11 +35,9 @@ object CommandLineParser {
 class CommandLineParser extends scopt.OptionParser[CommandLineArguments]("java -jar /path/to/cromwell.jar") {
 
   private def commonSubmissionArguments = List(
-    arg[String]("workflow-source").text("Workflow source file.").required().
-      action((s, c) => c.copy(workflowSource = Option(DefaultPathBuilder.get(s)))),
-    opt[String]('u',"workflow-url").text("Workflow source url.").
+    arg[String]("workflow-source").text("Workflow source file or workflow url.").required().
       action((s, c) =>
-        c.copy(workflowUrl = Option(s))),
+        c.copy(workflowSource = Option(s))),
     opt[String]("workflow-root").text("Workflow root.").
       action((s, c) =>
         c.copy(workflowRoot = Option(s))),

server/src/main/scala/cromwell/CromwellEntryPoint.scala

@@ -11,10 +11,11 @@ import com.typesafe.config.ConfigFactory
 import common.exception.MessageAggregation
 import common.validation.ErrorOr._
 import cromwell.CommandLineArguments.ValidSubmission
+import cromwell.CommandLineArguments.WorkflowSourceOrUrl
 import cromwell.CromwellApp._
 import cromwell.api.CromwellClient
 import cromwell.api.model.{Label, LabelsJsonFormatter, WorkflowSingleSubmission}
-import cromwell.core.path.Path
+import cromwell.core.path.{DefaultPathBuilder, Path}
 import cromwell.core.{WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports}
 import cromwell.engine.workflow.SingleWorkflowRunnerActor
 import cromwell.engine.workflow.SingleWorkflowRunnerActor.RunWorkflow
@@ -175,17 +176,24 @@ object CromwellEntryPoint extends GracefulStopSupport {
     import spray.json._
 
     val validation = args.validateSubmission(EntryPointLogger) map {
-      case ValidSubmission(w, u, r, i, o, l, z) =>
+      case ValidSubmission(w, u, r, i, o, l, z) =>{
+        val finalWorkflowSourceAndUrl: WorkflowSourceOrUrl = {
+          if (w.isDefined) WorkflowSourceOrUrl(w,u)  // submission has CWL workflow file path and no imports
+          else if (u.get.startsWith("http")) WorkflowSourceOrUrl(w, u)
+          else WorkflowSourceOrUrl(Option(DefaultPathBuilder.get(u.get).contentAsString), None) //case where url is a WDL/CWL file
+        }
+
         WorkflowSingleSubmission(
-          workflowSource = w,
-          workflowUrl = u,
+          workflowSource = finalWorkflowSourceAndUrl.source,
+          workflowUrl = finalWorkflowSourceAndUrl.url,
           workflowRoot = r,
           workflowType = args.workflowType,
           workflowTypeVersion = args.workflowTypeVersion,
           inputsJson = Option(i),
           options = Option(o),
           labels = Option(l.parseJson.convertTo[List[Label]]),
           zippedImports = z)
+      }
     }
 
     validOrFailSubmission(validation)

server/src/test/resources/cwl_glob_sort.cwl

@@ -0,0 +1,16 @@
+cwlVersion: v1.0
+class: CommandLineTool
+requirements:
+  - class: InlineJavascriptRequirement
+hints:
+  DockerRequirement:
+    dockerPull: "debian:stretch-slim"
+inputs: []
+baseCommand: [touch, z, y, x, w, c, b, a]
+outputs:
+  letters:
+    type: string
+    outputBinding:
+      glob: '?'
+      outputEval: |
+        ${ return self.sort(function(a,b) { return a.location > b.location ? 1 : (a.location < b.location ? -1 : 0) }).map(f => f.basename).join(" ") }