WM-2454: Private GitHub support on describe api

core/src/main/resources/reference.conf

@@ -418,7 +418,7 @@ docker {
       throttle {
         number-of-requests = 1000
         per = 60 seconds
-      }      
+      }
       num-threads = 10
     }
     google {
@@ -571,6 +571,16 @@ services {
     # Default - run within the Cromwell JVM
     class = "cromwell.services.womtool.impl.WomtoolServiceInCromwellActor"
   }
+  GithubAuthVending {
+    class = "cromwell.services.auth.impl.GithubAuthVendingActor"
+    config {
+      enabled = false
+      # Notes:
+      #   - don't include the 'Bearer' before the token
+      #   - this config value should be removed when support for fetching tokens from ECM has been added to Cromwell
+      access-token = "dummy-token"
+    }
+  }
 }
 
 include required(classpath("reference_database.inc.conf"))

engine/src/main/scala/cromwell/webservice/routes/WomtoolRouteSupport.scala

@@ -2,12 +2,14 @@
 
 import akka.actor.{ActorRef, ActorRefFactory}
 import akka.http.scaladsl.model._
+import akka.http.scaladsl.model.headers.OAuth2BearerToken
 import akka.http.scaladsl.server.Directives._
 import akka.http.scaladsl.server.Route
-import akka.pattern.ask
 import akka.stream.ActorMaterializer
 import akka.util.Timeout
 import cromwell.core.{WorkflowOptions, WorkflowSourceFilesCollection}
+import cromwell.languages.util.ImportResolver.ImportAuthProvider
+import cromwell.services.auth.GithubAuthVendingSupport
 import cromwell.services.womtool.WomtoolServiceMessages.{
   DescribeFailure,
   DescribeRequest,
@@ -20,7 +22,7 @@
 import scala.concurrent.ExecutionContext
 import scala.util.{Failure, Success}
 
-trait WomtoolRouteSupport extends WebServiceUtils {
+trait WomtoolRouteSupport extends WebServiceUtils with GithubAuthVendingSupport {
 
   implicit def actorRefFactory: ActorRefFactory
   implicit val ec: ExecutionContext
@@ -33,17 +35,25 @@
     path("womtool" / Segment / "describe") { _ =>
       post {
         entity(as[Multipart.FormData]) { formData: Multipart.FormData =>
-          onComplete(materializeFormData(formData)) {
-            case Success(data) =>
-              validateAndSubmitRequest(data)
-            case Failure(e) =>
-              e.failRequest(StatusCodes.InternalServerError)
+          extractCredentials { creds =>
+            val authProviders: List[ImportAuthProvider] = creds match {
+              case Some(OAuth2BearerToken(token)) => List(importAuthProvider(token))
+              case _ => List.empty
+            }
+            onComplete(materializeFormData(formData)(timeout, materializer, ec)) {
+              case Success(data) =>
+                validateAndSubmitRequest(data, authProviders)
+              case Failure(e) =>
+                e.failRequest(StatusCodes.InternalServerError)
+            }
           }
         }
       }
     }
 
-  private def validateAndSubmitRequest(data: MaterializedFormData): Route = {
+  private def validateAndSubmitRequest(data: MaterializedFormData,
+                                       importAuthProviders: List[ImportAuthProvider]
+  ): Route = {
     // TODO: move constants to WebServiceUtils, adopt in PartialWorkflowSources
     val workflowSource = data.get("workflowSource").map(_.utf8String)
     val workflowUrl = data.get("workflowUrl").map(_.utf8String)
@@ -66,7 +76,7 @@
       requestedWorkflowId = None
     )
 
-    onComplete(serviceRegistryActor.ask(DescribeRequest(wsfc)).mapTo[DescribeResult]) {
+    onComplete(serviceRegistryActor.ask(DescribeRequest(wsfc, importAuthProviders)).mapTo[DescribeResult]) {
       case Success(response: DescribeSuccess) =>
         import cromwell.services.womtool.models.WorkflowDescription.workflowDescriptionEncoder
         import de.heikoseeberger.akkahttpcirce.ErrorAccumulatingCirceSupport._

engine/src/test/scala/cromwell/webservice/routes/CromwellApiServiceSpec.scala

@@ -748,7 +748,7 @@ object CromwellApiServiceSpec {
             )
           case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh")
         }
-      case DescribeRequest(sourceFiles) =>
+      case DescribeRequest(sourceFiles, _) =>
         sourceFiles.workflowSource match {
           case Some("fail to describe") =>
             sender() ! DescribeFailure("as requested, failing to describe")

languageFactories/language-factory-core/src/main/scala/cromwell/languages/util/ImportResolver.scala

@@ -10,6 +10,7 @@
 import com.softwaremill.sttp._
 import com.softwaremill.sttp.asynchttpclient.cats.AsyncHttpClientCatsBackend
 import com.typesafe.config.ConfigFactory
+import com.typesafe.scalalogging.StrictLogging
 import common.Checked
 import common.transforms.CheckedAtoB
 import common.validation.ErrorOr._
@@ -22,11 +23,11 @@
 import java.security.MessageDigest
 import cromwell.core.WorkflowId
 import wom.ResolvedImportRecord
-import wom.core.WorkflowSource
+import wom.core.{WorkflowSource, WorkflowUrl}
 import wom.values._
 
 import scala.concurrent.duration._
-import scala.concurrent.{Await, ExecutionContext}
+import scala.concurrent.{Await, ExecutionContext, Future}
 import scala.util.{Failure, Success, Try}
 
 object ImportResolver {
@@ -37,6 +38,15 @@
                                   resolvedImportRecord: ResolvedImportRecord
   )
 
+  trait ImportAuthProvider {
+    def validHosts: List[String]
+    def authHeader(): Future[Map[String, String]]
+  }
+
+  trait GithubImportAuthProvider extends ImportAuthProvider {
+    override def validHosts: List[String] = List("github.com", "githubusercontent.com", "raw.githubusercontent.com")
+  }
+
   trait ImportResolver {
     def name: String
     protected def innerResolver(path: String, currentResolvers: List[ImportResolver]): Checked[ResolvedImportBundle]
@@ -179,20 +189,23 @@
     }
   }
 
-  case class HttpResolver(relativeTo: Option[String], headers: Map[String, String], hostAllowlist: Option[List[String]])
-      extends ImportResolver {
+  case class HttpResolver(relativeTo: Option[String],
+                          headers: Map[String, String],
+                          hostAllowlist: Option[List[String]],
+                          authProviders: List[ImportAuthProvider]
+  ) extends ImportResolver
+      with StrictLogging {
     import HttpResolver._
 
     override def name: String = relativeTo match {
       case Some(relativeToPath) => s"http importer (relative to $relativeToPath)"
       case None => "http importer (no 'relative-to' origin)"
-
     }
 
     def newResolverList(newRoot: String): List[ImportResolver] = {
       val rootWithoutFilename = newRoot.split('/').init.mkString("", "/", "/")
       List(
-        HttpResolver(relativeTo = Some(canonicalize(rootWithoutFilename)), headers, hostAllowlist)
+        HttpResolver(relativeTo = Some(canonicalize(rootWithoutFilename)), headers, hostAllowlist, authProviders)
       )
     }
 
@@ -211,8 +224,13 @@
       case None => true
     }
 
+    def fetchAuthHeaders(uri: Uri): Future[Map[String, String]] =
+      authProviders collectFirst {
+        case provider if provider.validHosts.contains(uri.host) => provider.authHeader()
+      } getOrElse Future.successful(Map.empty[String, String])
+
     override def innerResolver(str: String, currentResolvers: List[ImportResolver]): Checked[ResolvedImportBundle] =
-      pathToLookup(str) flatMap { toLookup: WorkflowSource =>
+      pathToLookup(str) flatMap { toLookup: WorkflowUrl =>
         (Try {
           val uri: Uri = uri"$toLookup"
 
@@ -227,21 +245,37 @@
         }).contextualizeErrors(s"download $toLookup")
       }
 
-    private def getUri(toLookup: WorkflowSource): Either[NonEmptyList[WorkflowSource], ResolvedImportBundle] = {
-      implicit val sttpBackend = HttpResolver.sttpBackend()
-      val responseIO: IO[Response[WorkflowSource]] = sttp.get(uri"$toLookup").headers(headers).send()
-
-      // temporary situation to get functionality working before
+    private def getUri(toLookup: WorkflowUrl): Checked[ResolvedImportBundle] = {
+      // Temporary situation to get functionality working before
       // starting in on async-ifying the entire WdlNamespace flow
-      val result: Checked[WorkflowSource] = Await.result(responseIO.unsafeToFuture(), 15.seconds).body.leftMap { e =>
-        NonEmptyList(e.toString.trim, List.empty)
+      // Note: this will cause the calling thread to block for up to 20 seconds
+      // (5 for the auth header lookup, 15 for the http request)
+      val unauthedAttempt = getUriInner(toLookup, Map.empty)
+      val result = if (unauthedAttempt.code == StatusCodes.NotFound) {
+        val authHeaders = Await.result(fetchAuthHeaders(uri"$toLookup"), 5.seconds)
+        if (authHeaders.nonEmpty) {
+          getUriInner(toLookup, authHeaders)
+        } else {
+          unauthedAttempt
+        }
+      } else {
+        unauthedAttempt
       }
 
-      result map {
+      result.body.leftMap { e =>
+        NonEmptyList.of(e.trim)
+      } map {
         ResolvedImportBundle(_, newResolverList(toLookup), ResolvedImportRecord(toLookup))
       }
     }
 
+    protected def getUriInner(toLookup: WorkflowUrl, authHeaders: Map[String, String]): Response[WorkflowSource] = {
+      implicit val sttpBackend = HttpResolver.sttpBackend()
+
+      val responseIO: IO[Response[WorkflowSource]] = sttp.get(uri"$toLookup").headers(headers ++ authHeaders).send()
+      Await.result(responseIO.unsafeToFuture(), 15.seconds)
+    }
+
     override def cleanupIfNecessary(): ErrorOr[Unit] = ().validNel
 
     override def hashKey: ErrorOr[String] = relativeTo.toString.md5Sum.validNel
@@ -252,15 +286,18 @@
     import common.util.IntrospectableLazy
     import common.util.IntrospectableLazy._
 
-    def apply(relativeTo: Option[String] = None, headers: Map[String, String] = Map.empty): HttpResolver = {
+    def apply(relativeTo: Option[String] = None,
+              headers: Map[String, String] = Map.empty,
+              authProviders: List[ImportAuthProvider] = List.empty
+    ): HttpResolver = {
       val config = ConfigFactory.load().getConfig("languages.WDL.http-allow-list")
       val allowListEnabled = config.as[Option[Boolean]]("enabled").getOrElse(false)
       val allowList: Option[List[String]] =
         if (allowListEnabled)
           config.as[Option[List[String]]]("allowed-http-hosts")
         else None
 
-      new HttpResolver(relativeTo, headers, allowList)
+      new HttpResolver(relativeTo, headers, allowList, authProviders)
     }
 
     val sttpBackend: IntrospectableLazy[SttpBackend[IO, Nothing]] = lazily {