diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 29a597bc13e..00000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,174 +0,0 @@ -version: 2.1 - -orbs: - slack: circleci/slack@4.2.1 - build-tools: circleci/build-tools@2.9.0 - -jobs: - test: - parameters: - machine-size: - type: string - default: large - build-type: - type: string - build-mysql: - type: string - default: "" - build-mariadb: - type: string - default: "" - build-postgresql: - type: string - default: "" - machine: - image: ubuntu-2004:202010-01 - resource_class: << parameters.machine-size >> - steps: - - run: - name: Halt builds except SBT test suite if there is no PR associated with the commit - command: | - if [[ -z "${CI_PULL_REQUEST}" ]] && [[ "${BUILD_TYPE}" != "sbt" ]] ; then - circleci-agent step halt - fi - - checkout - - run: - name: Custom step - configure GIT identity - command: | - git config user.email "circleci@example.com" - git config user.name "CircleCI" - - build-tools/merge-with-parent: - parent: develop - - restore_cache: - key: sbt-cache - - run: - command: src/ci/bin/test.sh - no_output_timeout: 1h - - run: - name: Do tricks to avoid unnecessary cache updates - command: | - find ~/.ivy2/cache -name "ivydata-*.properties" -print -delete - find ~/.sbt -name "*.lock" -print -delete - - store_test_results: - path: target/test-reports - - save_cache: - key: sbt-cache - paths: - - "~/.ivy2/cache" - - "~/.sbt" - environment: - CIRCLE_COMMIT_RANGE: << pipeline.git.base_revision >>...<< pipeline.git.revision >> - BUILD_TYPE: << parameters.build-type >> - BUILD_MYSQL: << parameters.build-mysql >> - BUILD_MARIADB: << parameters.build-mariadb >> - BUILD_POSTGRESQL: << parameters.build-postgresql >> - -workflows: - all-tests: - jobs: - - test: - name: testSbt - build-type: "sbt" - - test: - name: testSingleWorkflowRunner - build-type: "singleWorkflowRunner" - - test: - name: testDbms - build-type: "dbms" - - test: - name: testHoricromtalDeadlock - build-type: "horicromtalDeadlock" - - test: - name: testDockerScripts - build-type: "dockerScripts" - - test: - name: testReferenceDiskManifestBuilderApp - build-type: "referenceDiskManifestBuilderApp" - - test: - name: testCentaurAws - build-type: "centaurAws" - build-mysql: "5.7" - - test: - name: testCentaurDummy - build-type: "centaurDummy" - build-mysql: "5.7" - - test: - name: testCentaurEngineUpgradeLocal - build-type: "centaurEngineUpgradeLocal" - build-mysql: "5.7" - - test: - name: testCentaurEngineUpgradePapiV2alpha1 - build-type: "centaurEngineUpgradePapiV2alpha1" - build-mysql: "5.7" - - test: - name: testCentaurHoricromtalPapiV2alpha1 - build-type: "centaurHoricromtalPapiV2alpha1" - build-mysql: "5.7" - - test: - name: testCentaurHoricromtalPapiV2beta-MySQL - build-type: "centaurHoricromtalPapiV2beta" - build-mysql: "5.7" - - test: - name: testCentaurHoricromtalPapiV2beta-MariaDB - build-type: "centaurHoricromtalPapiV2beta" - build-mariadb: "10.3" - - test: - name: testCentaurHoricromtalEngineUpgradePapiV2alpha1-MySQL - build-type: "centaurHoricromtalEngineUpgradePapiV2alpha1" - build-mysql: "5.7" - - test: - name: testCentaurHoricromtalEngineUpgradePapiV2alpha1-MariaDB - build-type: "centaurHoricromtalEngineUpgradePapiV2alpha1" - build-mariadb: "10.3" - - test: - name: testCentaurPapiUpgradePapiV2alpha1 - build-type: "centaurPapiUpgradePapiV2alpha1" - build-mysql: "5.7" - - test: - name: testCentaurPapiUpgradeNewWorkflowsPapiV2alpha1 - build-type: "centaurPapiUpgradeNewWorkflowsPapiV2alpha1" - build-mysql: "5.7" - - test: - name: testCentaurLocal-MySQL - build-type: "centaurLocal" - build-mysql: "5.7" - - test: - name: testCentaurLocal-Postgresql - build-type: "centaurLocal" - build-postgresql: "11.3" - - test: - name: testCentaurPapiV2alpha1 - build-type: "centaurPapiV2alpha1" - build-mysql: "5.7" - - test: - name: testCentaurPapiV2beta - build-type: "centaurPapiV2beta" - build-mysql: "5.7" - - test: - name: testCentaurSlurm - build-type: "centaurSlurm" - build-mysql: "5.7" - - test: - name: testCentaurTes - build-type: "centaurTes" - build-mysql: "5.7" - - test: - name: testCentaurWdlUpgradeLocal - build-type: "centaurWdlUpgradeLocal" - build-mysql: "5.7" - - test: - name: testCheckPublish - build-type: "checkPublish" - build-mysql: "5.7" - - test: - name: testConformanceLocal - build-type: "conformanceLocal" - build-mysql: "5.7" - - test: - name: testConformancePapiV2beta - build-type: "conformancePapiV2beta" - build-mysql: "5.7" - - test: - name: testConformanceTesk - build-type: "conformanceTesk" - build-mysql: "5.7" diff --git a/.gitattributes b/.gitattributes index d4b49d9edc9..9fcf38aa0a7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,4 +3,3 @@ *.MD text *.java text *.html text -docs/api/RESTAPI.md linguist-generated=true diff --git a/.github/set_up_cromwell_action/action.yml b/.github/set_up_cromwell_action/action.yml new file mode 100644 index 00000000000..13cb11601db --- /dev/null +++ b/.github/set_up_cromwell_action/action.yml @@ -0,0 +1,47 @@ +#This is a series of steps that will setup Java/sbt/Cromwell on the local runner. +#These steps are meant to be re-used and invoked by other Github Action Workflows +name: 'Set Up Cromwell Steps' +description: Specific steps that will set up git secrets, java, sbt, and Cromwell on the local machine. +inputs: + cromwell_repo_token: #As an input to this action, you are required to pass in a token that can be used to authenticate while checking out Cromwell. + required: true + +runs: + using: "composite" # <-- this allows these steps to be used by other workflows. + steps: + #Allows this github action to use a cache to store stuff like Java and sbt files between runs. + - uses: actions/checkout@v3 + name: Checkout Coursier Cache + - uses: coursier/cache-action@v6 + name: Enable Coursier Cache + + #Cromwell requires git-secrets be setup. Here, we set up secrets and verify success with a script. + - name: Git secrets setup + run: | + git clone https://github.com/awslabs/git-secrets.git ~/git-secrets + cd ~/git-secrets + git checkout ad82d68ee924906a0401dfd48de5057731a9bc84 + sudo make install + shell: bash + + - name: Secrets check + run: | + sudo ln -s "$(which echo)" /usr/local/bin/say + ./minnie-kenny.sh --force + git secrets --scan-history + shell: bash + + #Clone the cromwell repo to this VM. + - name: Clone Cromwell + uses: actions/checkout@v3 + with: + repository: broadinstitute/cromwell + token: ${{ inputs.cromwell_repo_token }} + + #Install Java to this VM. This Java version and distribution is compatible with Cromwell. + - name: Setup JDK + uses: actions/setup-java@v3 + with: + distribution: temurin + java-version: 11 + diff --git a/.github/workflows/azure_e2e_run_workflow.yml b/.github/workflows/azure_e2e_run_workflow.yml new file mode 100644 index 00000000000..5071fc37e8b --- /dev/null +++ b/.github/workflows/azure_e2e_run_workflow.yml @@ -0,0 +1,117 @@ +name: 'Azure e2e - Run Workflow' +on: + schedule: + - cron: '0 16 * * *' # UTC 4pm, EST 11am, EDT 12pm + workflow_dispatch: + +env: + BROADBOT_TOKEN: '${{ secrets.BROADBOT_GITHUB_TOKEN }}' # github token for access to kick off a job in the private repo + RUN_NAME_SUFFIX: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}' + +jobs: + + # This job provisions useful parameters for e2e tests + params-gen: + runs-on: ubuntu-latest + permissions: + contents: 'read' + id-token: 'write' + outputs: + project-name: ${{ steps.gen.outputs.project_name }} + bee-name: '${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt}}-dev' + steps: + - name: Generate a random billing project name + id: 'gen' + run: | + project_name=$(echo "tmp-billing-project-$(uuidgen)" | cut -c -30) + echo "project_name=${project_name}" >> $GITHUB_OUTPUT + + create-bee-workflow: + runs-on: ubuntu-latest + needs: [params-gen] + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: Dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: bee-create + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + # NOTE: Opting to use "prod" instead of custom tag since I specifically want to test against the current prod state + # NOTE: For testing/development purposes I'm using dev + inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}", "version-template": "dev", "bee-template-name": "rawls-e2e-azure-tests"}' + + create-and-attach-billing-project-to-landing-zone-workflow: + runs-on: ubuntu-latest + needs: [create-bee-workflow, params-gen] + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: attach-billing-project-to-landing-zone.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ + "run-name": "attach-billing-project-to-landing-zone-${{ env.RUN_NAME_SUFFIX }}", + "bee-name": "${{ needs.params-gen.outputs.bee-name }}", + "billing-project": "${{ needs.params-gen.outputs.project-name }}", + "service-account": "firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com" }' + + run-cromwell-az-e2e: + needs: [params-gen, create-and-attach-billing-project-to-landing-zone-workflow] + permissions: + contents: read + id-token: write + uses: "broadinstitute/dsp-reusable-workflows/.github/workflows/cromwell-az-e2e-test.yaml@main" + with: + bee-name: "${{ needs.params-gen.outputs.bee-name }}" + billing-project-name: "${{ needs.params-gen.outputs.project-name }}" + + delete-billing-project-v2-from-bee-workflow: + continue-on-error: true + runs-on: ubuntu-latest + needs: [run-cromwell-az-e2e, create-and-attach-billing-project-to-landing-zone-workflow, params-gen] + if: always() + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: .github/workflows/delete-billing-project-v2-from-bee.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ + "run-name": "delete-billing-project-v2-from-bee-${{ env.RUN_NAME_SUFFIX }}", + "bee-name": "${{ needs.params-gen.outputs.bee-name }}", + "billing-project": "${{ needs.params-gen.outputs.project-name }}", + "service-account": "firecloud-qa@broad-dsde-qa.iam.gserviceaccount.com", + "silent-on-failure": "false" }' + + destroy-bee-workflow: + runs-on: ubuntu-latest + needs: [params-gen, create-bee-workflow, delete-billing-project-v2-from-bee-workflow] + if: always() + permissions: + contents: 'read' + id-token: 'write' + steps: + - name: dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v3 + with: + workflow: bee-destroy.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ env.BROADBOT_TOKEN }} + inputs: '{ "bee-name": "${{ needs.params-gen.outputs.bee-name }}" }' + wait-for-completion: true + + report-workflow: + uses: broadinstitute/sherlock/.github/workflows/client-report-workflow.yaml@main + with: + notify-slack-channels-upon-workflow-failure: "#cromwell_jenkins_ci_errors" + permissions: + id-token: write diff --git a/.github/workflows/chart_update_on_merge.yml b/.github/workflows/chart_update_on_merge.yml index 9e847bc17eb..a17dd971436 100644 --- a/.github/workflows/chart_update_on_merge.yml +++ b/.github/workflows/chart_update_on_merge.yml @@ -9,8 +9,14 @@ jobs: chart-update: name: Cromwhelm Chart Auto Updater if: github.event.pull_request.merged == true - runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners + runs-on: ubuntu-latest steps: + - name: Fetch Jira ID from the commit message + id: fetch-jira-id + run: | + JIRA_ID=$(echo '${{ github.event.pull_request.title }}' | grep -Eo '[A-Z][A-Z]+-[0-9]+' | xargs echo -n | tr '[:space:]' ',') + [[ -z "$JIRA_ID" ]] && { echo "No Jira ID found in $1" ; exit 1; } + echo "JIRA_ID=$JIRA_ID" >> $GITHUB_OUTPUT - name: Clone Cromwell uses: actions/checkout@v2 with: @@ -41,20 +47,35 @@ jobs: echo "CROMWELL_NUMBER=$((previous_version + 1))" >> $GITHUB_ENV - name: Save complete image ID run: | - echo "CROMWELL_SNAP_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA-SNAP"`" >> $GITHUB_ENV + echo "CROMWELL_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA"`" >> $GITHUB_ENV # `DSDEJENKINS_PASSWORD` auto syncs from vault with https://github.com/broadinstitute/terraform-ap-deployments/pull/614 - name: Login to Docker Hub uses: docker/login-action@v1 with: username: dsdejenkins password: ${{ secrets.DSDEJENKINS_PASSWORD }} + # Build & push `cromwell`, `womtool`, `cromiam`, and `cromwell-drs-localizer` + # This step is validated in the GHA 'docker_build_test.yml' without the accompanying docker push - name: Build Cromwell Docker run: | set -e cd cromwell - sbt server/docker - docker push broadinstitute/cromwell:$CROMWELL_SNAP_VERSION - - name: Edit & push chart + sbt -Dproject.isSnapshot=false dockerBuildAndPush + - name: Deploy to dev and board release train (Cromwell) + uses: broadinstitute/repository-dispatch@master + with: + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + repository: broadinstitute/terra-helmfile + event-type: update-service + client-payload: '{"service": "cromwell", "version": "${{ env.CROMWELL_VERSION }}", "dev_only": false}' + - name: Deploy to dev and board release train (CromIAM) + uses: broadinstitute/repository-dispatch@master + with: + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + repository: broadinstitute/terra-helmfile + event-type: update-service + client-payload: '{"service": "cromiam", "version": "${{ env.CROMWELL_VERSION }}", "dev_only": false}' + - name: Edit & push cromwhelm chart env: BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} run: | @@ -62,10 +83,53 @@ jobs: cd cromwhelm git checkout main ls -la - sed -i "s/appVersion.*/appVersion: \"$CROMWELL_SNAP_VERSION\"/" cromwell-helm/Chart.yaml - sed -i "s/image: broadinstitute\/cromwell.*/image: broadinstitute\/cromwell:$CROMWELL_SNAP_VERSION/" cromwell-helm/templates/cromwell.yaml + sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" terra-batch-libchart/values.yaml + git diff git config --global user.name "broadbot" git config --global user.email "broadbot@broadinstitute.org" - git commit -am "Auto update to Cromwell $CROMWELL_SNAP_VERSION" + git commit -am "${{ steps.fetch-jira-id.outputs.JIRA_ID }}: Auto update to Cromwell $CROMWELL_VERSION" git push https://broadbot:$BROADBOT_GITHUB_TOKEN@github.com/broadinstitute/cromwhelm.git main + cd - + + - name: Clone terra-helmfile + uses: actions/checkout@v3 + with: + repository: broadinstitute/terra-helmfile + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # Has to be set at checkout AND later when pushing to work + path: terra-helmfile + + - name: Update workflows-app in terra-helmfile + run: | + set -e + cd terra-helmfile + sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/workflows-app/values.yaml + cd - + + - name: Update cromwell-runner-app in terra-helmfile + run: | + set -e + cd terra-helmfile + sed -i "s|image: broadinstitute/cromwell:.*|image: broadinstitute/cromwell:$CROMWELL_VERSION|" charts/cromwell-runner-app/values.yaml + cd - + + + - name: Make PR in terra-helmfile + env: + BROADBOT_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + GH_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + run: | + set -e + JIRA_ID=${{ steps.fetch-jira-id.outputs.JIRA_ID }} + if [[ $JIRA_ID == "missing" ]]; then + echo "JIRA_ID missing, PR to terra-helmfile will not be created" + exit 0; + fi + cd terra-helmfile + git checkout -b ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION + git config --global user.name "broadbot" + git config --global user.email "broadbot@broadinstitute.org" + git commit -am "${JIRA_ID}: Auto update Cromwell to $CROMWELL_VERSION in workflows-app and cromwell-runner-app" + git push -u origin ${JIRA_ID}-cromwell-update-$CROMWELL_VERSION + gh pr create --title "${JIRA_ID}: auto update Cromwell version to $CROMWELL_VERSION in workflows-app and cromwell-runner-app" --body "${JIRA_ID} helm chart update" --label "automerge" + cd - diff --git a/.github/workflows/consumer_contract_tests.yml b/.github/workflows/consumer_contract_tests.yml new file mode 100644 index 00000000000..4635abaa765 --- /dev/null +++ b/.github/workflows/consumer_contract_tests.yml @@ -0,0 +1,200 @@ +name: Consumer contract tests +# The purpose of this workflow is to run a suite of Cromwell contract tests against mock service provider(s) using Pact framework. +# +# More details about Contract Testing can be found in our handbook +# +# https://broadworkbench.atlassian.net/wiki/spaces/IRT/pages/2660368406/Getting+Started+with+Pact+Contract+Testing +# +# This workflow involves Cromwell as a consumer, and ANY provider (e.g. Sam) Cromwell consumes. +# Each party owns a set of tests (aka contract tests). +# +# Consumer contract tests (aka consumer tests) runs on a mock provider service and does not require a real provider service. +# Provider contract tests (aka provider verification tests) runs independently of any consumer. +# +# Specifically: +# Cromwell runs consumer tests against mock service. Upon success, publish consumer pacts to +# Pact Broker https://pact-broker.dsp-eng-tools.broadinstitute.org/. +# +# Pact Broker is the source of truth to forge contractual obligations between consumer and provider. +# +# This workflow meets the criteria of Pact Broker *Platinum* as described in https://docs.pact.io/pact_nirvana/step_6. +# The can-i-deploy job has been added to this workflow to support *Platinum* and gate the code for promotion to default branch. +# +# This is how it works. +# +# Consumer makes a change that results in a new pact published to Pact Broker. +# Pact Broker notifies provider(s) of the changed pact and trigger corresponding verification workflows. +# Provider downloads relevant versions of consumer pacts from Pact Broker and kicks off verification tests against the consumer pacts. +# Provider updates Pact Broker with verification status. +# Consumer kicks off can-i-deploy on process to determine if changes can be promoted and used for deployment. +# +# NOTE: The publish-contracts workflow will use the latest commit of the branch that triggers this workflow to publish the unique consumer contract version to Pact Broker. +on: + pull_request: + branches: + - develop + paths-ignore: + - 'README.md' + push: + branches: + - develop + paths-ignore: + - 'README.md' + merge_group: + branches: + - develop + +env: + PUBLISH_CONTRACTS_RUN_NAME: 'publish-contracts-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}' + CAN_I_DEPLOY_RUN_NAME: 'can-i-deploy-${{ github.event.repository.name }}-${{ github.run_id }}-${{ github.run_attempt }}' + +jobs: + init-github-context: + runs-on: ubuntu-latest + outputs: + repo-branch: ${{ steps.extract-branch.outputs.repo-branch }} + repo-version: ${{ steps.extract-branch.outputs.repo-version }} + fork: ${{ steps.extract-branch.outputs.fork }} + + steps: + - uses: actions/checkout@v3 + + # Construct a version string like `87-9c6c439`. Adapted from `chart_update_on_merge.yml`. + - name: Find Cromwell short SHA + run: | + set -e + echo "CROMWELL_SHORT_SHA=`git rev-parse --short $GITHUB_SHA`" >> $GITHUB_ENV + - name: Find Cromwell release number + run: | + set -e + previous_version=$(curl -X GET https://api.github.com/repos/broadinstitute/cromwell/releases/latest | jq .tag_name | xargs) + if ! [[ "${previous_version}" =~ ^[0-9][0-9]+$ ]]; then + exit 1 + fi + echo "CROMWELL_NUMBER=$((previous_version + 1))" >> $GITHUB_ENV + - name: Save complete image ID + run: | + echo "CROMWELL_VERSION=`echo "$CROMWELL_NUMBER-$CROMWELL_SHORT_SHA"`" >> $GITHUB_ENV + + - name: Extract branch + id: extract-branch + run: | + GITHUB_EVENT_NAME=${{ github.event_name }} + if [[ "$GITHUB_EVENT_NAME" == "push" ]]; then + GITHUB_REF=${{ github.ref }} + GITHUB_SHA=${{ github.sha }} + elif [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then + GITHUB_REF=refs/heads/${{ github.head_ref }} + GITHUB_SHA=${{ github.event.pull_request.head.sha }} + elif [[ "$GITHUB_EVENT_NAME" == "merge_group" ]]; then + GITHUB_REF=refs/heads/${{ github.head_ref }} + else + echo "Failed to extract branch information" + exit 1 + fi + echo "CURRENT_BRANCH=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_ENV + echo "CURRENT_SHA=$GITHUB_SHA" >> $GITHUB_ENV + + echo "repo-branch=${GITHUB_REF/refs\/heads\//""}" >> $GITHUB_OUTPUT + echo "repo-version=${CROMWELL_VERSION}" >> $GITHUB_OUTPUT + echo "fork=${FORK}" >> $GITHUB_OUTPUT + + - name: Is PR triggered by forked repo? + if: ${{ steps.extract-branch.outputs.fork == 'true' }} + run: | + echo "PR was triggered by forked repo" + + - name: Echo repo and branch information + run: | + echo "repo-owner=${{ github.repository_owner }}" + echo "repo-name=${{ github.event.repository.name }}" + echo "repo-branch=${{ steps.extract-branch.outputs.repo-branch }}" + echo "repo-version=${{ steps.extract-branch.outputs.repo-version }}" + echo "fork=${{ steps.extract-branch.outputs.fork }}" + + cromwell-contract-tests: + runs-on: ubuntu-latest + needs: [init-github-context] + outputs: + pact-b64-drshub: ${{ steps.encode-pact.outputs.pact-b64-drshub }} + pact-b64-cbas: ${{ steps.encode-pact.outputs.pact-b64-cbas }} + + steps: + - uses: actions/checkout@v3 + - name: Run consumer tests + run: | + docker run --rm -v $PWD:/working \ + -v jar-cache:/root/.ivy \ + -v jar-cache:/root/.ivy2 \ + -w /working \ + sbtscala/scala-sbt:openjdk-17.0.2_1.7.2_2.13.10 \ + sbt "project pact4s" clean test + + - name: Output consumer contract as non-breaking base64 string + id: encode-pact + run: | + set -e + cd pact4s + NON_BREAKING_B64_DRSHUB=$(cat target/pacts/cromwell-drshub.json | base64 -w 0) + NON_BREAKING_B64_CBAS=$(cat target/pacts/cromwell-cbas.json | base64 -w 0) + echo "pact-b64-drshub=${NON_BREAKING_B64_DRSHUB}" >> $GITHUB_OUTPUT + echo "pact-b64-cbas=${NON_BREAKING_B64_CBAS}" >> $GITHUB_OUTPUT + + # Prevent untrusted sources from using PRs to publish contracts + # since access to secrets is not allowed. + publish-contracts: + runs-on: ubuntu-latest + if: ${{ needs.init-github-context.outputs.fork == 'false' || needs.init-github-context.outputs.fork == ''}} + needs: [init-github-context, cromwell-contract-tests] + steps: + - name: Dispatch drshub to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v4.0.0 + with: + run-name: "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}" + workflow: .github/workflows/publish-contracts.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo + inputs: '{ + "run-name": "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}", + "pact-b64": "${{ needs.cromwell-contract-tests.outputs.pact-b64-drshub }}", + "repo-owner": "${{ github.repository_owner }}", + "repo-name": "${{ github.event.repository.name }}", + "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}", + "release-tag": "${{ needs.init-github-context.outputs.repo-version }}" + }' + - name: Dispatch cbas to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v4.0.0 + with: + run-name: "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}" + workflow: .github/workflows/publish-contracts.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo + inputs: '{ + "run-name": "${{ env.PUBLISH_CONTRACTS_RUN_NAME }}", + "pact-b64": "${{ needs.cromwell-contract-tests.outputs.pact-b64-cbas }}", + "repo-owner": "${{ github.repository_owner }}", + "repo-name": "${{ github.event.repository.name }}", + "repo-branch": "${{ needs.init-github-context.outputs.repo-branch }}", + "release-tag": "${{ needs.init-github-context.outputs.repo-version }}" + }' + + can-i-deploy: + runs-on: ubuntu-latest + if: ${{ needs.init-github-context.outputs.fork == 'false' || needs.init-github-context.outputs.fork == ''}} + needs: [ init-github-context, publish-contracts ] + steps: + - name: Dispatch to terra-github-workflows + uses: broadinstitute/workflow-dispatch@v4.0.0 + with: + run-name: "${{ env.CAN_I_DEPLOY_RUN_NAME }}" + workflow: .github/workflows/can-i-deploy.yaml + repo: broadinstitute/terra-github-workflows + ref: refs/heads/main + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} # github token for access to kick off a job in the private repo + inputs: '{ + "run-name": "${{ env.CAN_I_DEPLOY_RUN_NAME }}", + "pacticipant": "cromwell", + "version": "${{ needs.init-github-context.outputs.repo-version }}" + }' diff --git a/.github/workflows/cromwell_unit_tests.yml b/.github/workflows/cromwell_unit_tests.yml new file mode 100644 index 00000000000..88951871d8f --- /dev/null +++ b/.github/workflows/cromwell_unit_tests.yml @@ -0,0 +1,37 @@ +name: 'Cromwell unit tests' + +#This github action runs all of Cromwell's unit tests. + +#This is what shows up in the github workflows page as the title. +run-name: ${{ github.actor }} running Cromwell sbt unit tests. + +#What will trigger the workflow to run. +on: + workflow_dispatch: #Manual trigger from GitHub UI + push: + merge_group: + +permissions: + contents: read + +jobs: + build-and-test: + #This action is using a Github free runner, rather than a Broad self-hosted one. + #This is because the Broad ones don't have sbt installed by default. + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 # checkout the cromwell repo + - uses: ./.github/set_up_cromwell_action #Exectute this reusable github action. It will set up java/sbt/git-secrets/cromwell. + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + + #Invoke SBT to run all unit tests for Cromwell. + - name: Run tests + env: + AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }} + run: | + set -e + sbt "test" diff --git a/.github/workflows/docker_build_test.yml b/.github/workflows/docker_build_test.yml new file mode 100644 index 00000000000..b4d373f330e --- /dev/null +++ b/.github/workflows/docker_build_test.yml @@ -0,0 +1,43 @@ +name: 'Docker Build Test' + +# This test verifies that we can successfully build the same docker images that we release. +# Includes `cromwell`, `womtool`, `cromiam`, and `cromwell-drs-localizer` +# See chart_update_on_merge.yml for the actual release workflow. + +run-name: ${{ github.actor }} Docker Build Test + +on: + workflow_dispatch: + push: + merge_group: + +permissions: + contents: read + +jobs: + sbt-build: + name: sbt docker build + runs-on: ubuntu-latest + steps: + - name: Clone Cromwell + uses: actions/checkout@v2 + with: + repository: broadinstitute/cromwell + token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + path: cromwell + - uses: olafurpg/setup-scala@v10 + with: + java-version: adopt@1.11 + # The following invocation should be as similar as possible to the one in chart_update_on_merge.yml + # To state the obvious: This test should not publish anything. It should simply verify that the build completes. + - name: Build Cromwell Docker + run: | + set -e + cd cromwell + sbt -Dproject.isSnapshot=false docker + # Rarely used but we really want it always working for emergencies + - name: Build Cromwell Debug Docker + run: | + set -e + cd cromwell + sbt -Dproject.isDebug=true docker diff --git a/.github/workflows/integration_tests.yml b/.github/workflows/integration_tests.yml new file mode 100644 index 00000000000..5ff831a4b23 --- /dev/null +++ b/.github/workflows/integration_tests.yml @@ -0,0 +1,130 @@ +name: 'Integration Tests' + +#This github action runs all of Cromwell's integration tests. + +# This is what shows up in the github workflows page as the title. Using github ternary syntax & format() function. +run-name: ${{ github.event_name == 'schedule' && 'Nightly Integration Testing' || format('{0} Integration Testing', github.actor) }} + +#What will trigger the workflow to run. +on: + workflow_dispatch: #Manual trigger from GitHub UI + push: + schedule: + - cron: '0 0 * * 1-5' + merge_group: + +permissions: + contents: read + +concurrency: + # Don't run this workflow concurrently on the same branch + group: ${{ github.workflow }}-${{ github.ref }} + # For PRs, don't wait for completion of existing runs, cancel them instead + cancel-in-progress: ${{ github.ref != 'develop' }} + +jobs: + integration-tests: + strategy: + fail-fast: false #disabling fail-fast means that even if one test fails, the others will still try to complete. + #Each entry below is a single integration test that lives in /src/ci/bin/. + #Each will be launched on its own runner so they can occur in parallel. + #Friendly names are displayed on the Github UI and aren't used anywhere else. + matrix: + # Batch test fixes to land later + include: + - build_type: centaurGcpBatch + build_mysql: 5.7 + friendly_name: Centaur GCP Batch with MySQL 5.7 + - build_type: centaurPapiV2beta + build_mysql: 5.7 + friendly_name: Centaur Papi V2 Beta with MySQL 5.7 + - build_type: centaurPapiV2betaRestart + build_mysql: 5.7 + friendly_name: Centaur Papi V2 Beta (restart) + - build_type: dbms + friendly_name: DBMS + - build_type: centaurTes + build_mysql: 5.7 + friendly_name: Centaur TES with MySQL 5.7 + - build_type: centaurLocal + build_mysql: 5.7 + friendly_name: Centaur Local with MySQL 5.7 + - build_type: checkPublish + friendly_name: Check Publish + - build_type: centaurAws + build_mysql: 5.7 + friendly_name: Centaur AWS with MySQL 5.7 + - build_type: centaurDummy + build_mysql: 5.7 + friendly_name: Centaur Dummy with MySQL 5.7 + - build_type: centaurHoricromtalPapiV2beta + build_mysql: 5.7 + friendly_name: Centaur Horicromtal PapiV2 Beta with MySQL 5.7 + - build_type: horicromtalDeadlock + friendly_name: Horicromtal Deadlock + - build_type: singleWorkflowRunner + friendly_name: Single Workflow Runner + - build_type: centaurLocal + build_mariadb: 10.3 + friendly_name: Centaur Local with MariaDB 10.3 + - build_type: centaurLocal + build_postgresql: 11.3 + friendly_name: Centaur Local with PostgreSQL 11.3 + - build_type: centaurEngineUpgradeLocal + build_mysql: 5.7 + friendly_name: Centaur Engine Upgrade Local with MySQL 5.7 + - build_type: referenceDiskManifestBuilderApp + friendly_name: Reference Disk Manifest Builder App + - build_type: centaurSlurm + build_mysql: 5.7 + friendly_name: "Centaur Slurm with MySQL 5.7" + - build_type: centaurBlob + build_mysql: 5.7 + friendly_name: Centaur Blob + name: ${{ matrix.friendly_name }} + env: + BUILD_NAME: ${{ matrix.build_type }} + BUILD_TYPE: ${{ matrix.build_type }} #intentionally duplicated variable + BUILD_MYSQL: ${{ matrix.build_mysql }} + BUILD_POSTGRESQL: ${{ matrix.build_postgresql }} + BUILD_MARIADB: ${{ matrix.build_mariadb }} + VAULT_ROLE_ID: ${{ secrets.VAULT_ROLE_ID_CI }} + VAULT_SECRET_ID: ${{ secrets.VAULT_SECRET_ID_CI }} + AZURE_CLIENT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_ID }} + AZURE_CLIENT_SECRET: ${{ secrets.VAULT_AZURE_CENTAUR_CLIENT_SECRET }} + AZURE_TENANT_ID: ${{ secrets.VAULT_AZURE_CENTAUR_TENANT_ID }} + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + - uses: actions/checkout@v3 # checkout the cromwell repo + with: + ref: ${{ inputs.target-branch }} + - uses: ./.github/set_up_cromwell_action #This github action will set up git-secrets, caching, java, and sbt. + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + #This script bascially just looks up another script to run, assuming that the other script's filename is: + #src/ci/bin/test${BUILD_TYPE}.sh. The first letter of the BUILD_TYPE is automatically capitalized when looking. + - name: Run Integration Test + shell: 'script -q -e -c "bash --noprofile --norc -eo pipefail {0}"' #See comment below + run: | + set -e + echo Running test.sh + ./src/ci/bin/test.sh + - uses: codecov/codecov-action@v4 + with: + token: ${{ secrets.CODECOV_TOKEN }} + fail_ci_if_error: false # Tolerate missing codecov reports, since not all suites generate them. + # always() is some github magic that forces the following step to run, even when the previous fails. + # Without it, the if statement won't be evaluated on a test failure. + - uses: ravsamhq/notify-slack-action@v2 + if: always() && github.ref == 'refs/heads/develop' #only report on failures against develop. + with: + status: ${{ job.status }} + notify_when: "failure" + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + #The "shell: ..."" line is a way to force the Github Action Runner to use a bash shell that thinks it has a TTY. + #The issue and solution are described here: https://github.com/actions/runner/issues/241#issuecomment-842566950 + #This is only needed for ReferenceDiskManifestBuilderApp test. + #This test uses fancy colors in the output, which likely causes the problem. + #See WX-938. diff --git a/.github/workflows/make_publish_prs.yml b/.github/workflows/make_publish_prs.yml index ced80154c80..e4e98a7f2f0 100644 --- a/.github/workflows/make_publish_prs.yml +++ b/.github/workflows/make_publish_prs.yml @@ -16,7 +16,7 @@ on: jobs: make-firecloud-develop-pr: name: Create firecloud-develop PR - runs-on: self-hosted # Faster machines; see https://github.com/broadinstitute/cromwell/settings/actions/runners + runs-on: ubuntu-latest steps: - name: Clone firecloud-develop uses: actions/checkout@v2 @@ -50,7 +50,7 @@ jobs: git config --global user.email "broadbot@broadinstitute.org" git commit -m "Updating Cromwell version to ${NEW_CROMWELL_V}" git push https://broadbot:$BROADBOT_GITHUB_TOKEN@github.com/broadinstitute/firecloud-develop.git ${NEW_BRANCH_NAME} - echo ::set-output name=NEW_BRANCH_NAME::${NEW_BRANCH_NAME} + echo "NEW_BRANCH_NAME=${NEW_BRANCH_NAME}" >> $GITHUB_OUTPUT - name: Create firecloud-develop PR uses: actions/github-script@v6 with: @@ -70,4 +70,3 @@ jobs: 'It updates cromwell from version ${{ github.event.inputs.old_cromwell_version }} to ${{ github.event.inputs.new_cromwell_version }}.' ].join('\n') }); - diff --git a/.github/workflows/scalafmt-check.yml b/.github/workflows/scalafmt-check.yml new file mode 100644 index 00000000000..3730d2ffc8f --- /dev/null +++ b/.github/workflows/scalafmt-check.yml @@ -0,0 +1,31 @@ +name: 'ScalaFmt Check' + +# This GitHub Action runs the ScalaFmt linting tool on the entire codebase. +# It fails if any files are not formatted properly. +# If it is triggered by someone commenting 'scalafmt' on a PR, it will first format, commit, and push formatted code +# to the branch. + +run-name: ${{ format('ScalaFmt Check on {0}', github.ref_name) }} + +on: + workflow_dispatch: + push: + +permissions: + contents: read + +jobs: + run-scalafmt-check: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v3 + with: + ref: ${{ inputs.target-branch }} + - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + - name: Run ScalaFmt + run: | + sbt scalafmtCheckAll + working-directory: ${{ github.workspace }} diff --git a/.github/workflows/scalafmt-fix.yml b/.github/workflows/scalafmt-fix.yml new file mode 100644 index 00000000000..a20eab6dbee --- /dev/null +++ b/.github/workflows/scalafmt-fix.yml @@ -0,0 +1,64 @@ +name: 'ScalaFmt Fix' + +# This GitHub Action runs the ScalaFmt linting tool on the entire codebase. +# It will fix, commit, and push linted code. +# It will only run when someone comments "scalafmt" on a PR. + +run-name: ScalaFmt Fix + +on: + issue_comment: + types: + - created + workflow_dispatch: + branch_name: + description: 'Branch to run ScalaFmt against' + required: true + pull_request_target: + types: + - opened + - synchronize + +jobs: + run-scalafmt-fix: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - name: Determine Target Branch + id: determine-branch + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "::set-output name=target_branch::${{ inputs.branch_name }}" + else + echo "::set-output name=target_branch::${{ github.event.pull_request.head.ref }}" + fi + shell: bash + env: + inputs.branch_name: ${{ inputs.branch_name }} + - name: Check for ScalaFmt Comment + id: check-comment + run: | + if [[ "${{ github.event_name }}" == "issue_comment" && "${{ github.event.comment.body }}" == *"scalafmt"* ]]; then + echo "::set-output name=comment-triggered::true" + else + echo "::set-output name=comment-triggered::false" + fi + shell: bash + - uses: actions/checkout@v3 # checkout the cromwell repo + with: + ref: ${{ inputs.target-branch }} + - uses: ./.github/set_up_cromwell_action + with: + cromwell_repo_token: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + - name: Run ScalaFmt Fixup + if: steps.check-comment.outputs.comment-triggered == 'true' || github.event_name == 'workflow_dispatch' + env: + BROADBOT_GITHUB_TOKEN: ${{ secrets.BROADBOT_GITHUB_TOKEN }} + run: | + sbt scalafmtAll + git config --global user.email "broadbot@broadinstitute.org" + git config --global user.name "Broad Bot" + git add . + git commit -m "ScalaFmt fixup via Broad Bot" + git push origin ${{ steps.determine-branch.outputs.target_branch }} + working-directory: ${{ github.workspace }} diff --git a/.github/workflows/trivy.yml b/.github/workflows/trivy.yml index 0590d48ef53..d353eae4e58 100644 --- a/.github/workflows/trivy.yml +++ b/.github/workflows/trivy.yml @@ -12,7 +12,6 @@ jobs: project: - cromiam - cromwell-drs-localizer - - perf - server - womtool @@ -46,7 +45,7 @@ jobs: # export image name from the log image=$(grep 'Tagging image' build.log | awk '{print $NF}') - echo "::set-output name=image::${image}" + echo "image=${image}" >> $GITHUB_OUTPUT # scan the image - uses: broadinstitute/dsp-appsec-trivy-action@v1 diff --git a/.github/workflows/validate_pr_name.yml b/.github/workflows/validate_pr_name.yml new file mode 100644 index 00000000000..db26bbd95c6 --- /dev/null +++ b/.github/workflows/validate_pr_name.yml @@ -0,0 +1,23 @@ +# A github action to validate the name of a pull request contains a Jira tag: + +name: Validate PR name + +on: + pull_request: + types: [opened, edited, synchronize] + +jobs: + validate_pr_name: + runs-on: ubuntu-latest + steps: + - name: Validate PR title + id: validate + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const title = context.payload.pull_request.title; + const regex = /[A-Z][A-Z]+-\d+/; + if (!regex.test(title)) { + core.setFailed("PR title must contain a Jira tag"); + } diff --git a/.gitignore b/.gitignore index 571a12c5873..1a437ca4fc2 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # common scala config *~ +.metals/ .DS_Store .artifactory .bsp @@ -17,6 +18,15 @@ tags target /site +#from running integration tests locally +actual.json +console_output.txt +expected.json +run_mode_metadata.json + +#bloop files +/.bloop + # custom config cromwell-executions cromwell-test-executions @@ -38,7 +48,6 @@ cromwell-service-account.json cwl_conformance_test.inputs.json dockerhub_provider_config_v1.inc.conf dockerhub_provider_config_v2.inc.conf -github_private_deploy_key papi_application.inc.conf papi_refresh_token.options.json papi_v2_gcsa.options.json @@ -50,3 +59,10 @@ tesk_application.conf **/venv/ exome_germline_single_sample_v1.3/ **/*.pyc +src/ci/resources/*.temp + +# GHA credentials +gha-creds-*.json + +# jenv +.java-version diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 00000000000..376dcc3a901 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,14 @@ +# Read the Docs configuration file for MkDocs projects +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +# Set the version of Python and other tools you might need +build: + os: ubuntu-22.04 + tools: + python: "3.12" + +mkdocs: + configuration: mkdocs.yml diff --git a/.scalafmt.conf b/.scalafmt.conf new file mode 100644 index 00000000000..336b0fd7145 --- /dev/null +++ b/.scalafmt.conf @@ -0,0 +1,18 @@ +version = 3.7.17 +align.preset = none +align.openParenCallSite = true +align.openParenDefnSite = true +maxColumn = 120 +continuationIndent.defnSite = 2 +assumeStandardLibraryStripMargin = true +align.stripMargin = true +danglingParentheses.preset = true +rewrite.rules = [Imports, RedundantBraces, RedundantParens, SortModifiers] +rewrite.imports.sort = scalastyle +docstrings.style = keep +project.excludeFilters = [ + Dependencies.scala, + Settings.scala, + build.sbt +] +runner.dialect = scala213 diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index d733b16432c..00000000000 --- a/.travis.yml +++ /dev/null @@ -1,109 +0,0 @@ -os: linux -dist: focal -services: - - docker -language: minimal -git: - depth: false -cache: - directories: - - $HOME/.ivy2/cache - - $HOME/.coursier/cache - # see cromwell::private::delete_sbt_boot for more info - #- $HOME/.sbt/boot/ -before_cache: - # Tricks to avoid unnecessary cache updates - - find $HOME/.ivy2/cache -name "ivydata-*.properties" -print -delete - - find $HOME/.coursier/cache -name "ivydata-*.properties" -print -delete - - find $HOME/.sbt -name "*.lock" -print -delete -env: - jobs: - # Setting this variable twice will cause the 'script' section to run twice with the respective env var invoked - - >- - BUILD_TYPE=centaurAws - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurDummy - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurEngineUpgradeLocal - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurEngineUpgradePapiV2alpha1 - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurHoricromtalPapiV2alpha1 - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurHoricromtalPapiV2beta - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurHoricromtalEngineUpgradePapiV2alpha1 - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurPapiUpgradePapiV2alpha1 - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurPapiUpgradeNewWorkflowsPapiV2alpha1 - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurLocal - BUILD_MARIADB=10.3 - - >- - BUILD_TYPE=centaurLocal - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurLocal - BUILD_POSTGRESQL=11.3 - - >- - BUILD_TYPE=centaurPapiV2alpha1 - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurPapiV2beta - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurSlurm - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurTes - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=centaurWdlUpgradeLocal - BUILD_MYSQL=5.7 - - >- - BUILD_TYPE=checkPublish - - >- - BUILD_TYPE=horicromtalDeadlock - - >- - BUILD_TYPE=dockerScripts - - >- - BUILD_TYPE=sbt - BUILD_SBT_INCLUDE=engine - - >- - BUILD_TYPE=sbt - BUILD_SBT_INCLUDE=server - - >- - BUILD_TYPE=sbt - BUILD_SBT_INCLUDE=services - - >- - BUILD_TYPE=sbt - BUILD_SBT_EXCLUDE='engine|server|services' - - >- - BUILD_TYPE=dbms - - >- - BUILD_TYPE=singleWorkflowRunner - - >- - BUILD_TYPE=metadataComparisonPython - - >- - BUILD_TYPE=referenceDiskManifestBuilderApp -script: - - src/ci/bin/test.sh -notifications: - slack: - rooms: - - secure: B5KYcnhk/ujAUWlHsjzP7ROLm6MtYhaGikdYf6JYINovhMbVKnZCTlZEy7rqT3L2T5uJ25iefD500VQGk1Gn7puQ1sNq50wqjzQaj20PWEiBwoWalcV/nKBcQx1TyFT13LJv8fbFnVPxFCkC3YXoHedx8qAhDs8GH/tT5J8XOC8= - template: - - "Build <%{build_url}|#%{build_number}> (<%{compare_url}|%{commit}>) of %{repository}@%{branch} by %{author} %{result} in %{duration}" - on_success: change - on_failure: change - on_pull_requests: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 0c40f47b651..aecc6361de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,136 @@ # Cromwell Change Log +## 87 Release Notes + +### `upgrade` command removed from Womtool + +Womtool previously supported a `womtool upgrade` command for upgrading draft-2 WDLs to 1.0. With WDL 1.1 soon to become the latest supported version, this functionality is retiring. + +### Replacement of `gsutil` with `gcloud storage` + +In this release, all **localization** functionality on the GCP backend migrates to use the more modern and performant `gcloud storage`. With sufficiently powerful worker VMs, Cromwell can now localize at up to 1200 MB/s [0][1][2]. + +In a future release, **delocalization** will also migrate to `gcloud storage`. As part of that upcoming change, we are considering turning on [parallel composite uploads](https://cromwell.readthedocs.io/en/stable/backends/Google/#parallel-composite-uploads) by default to maximize performance. Delocalized composite objects will no longer have an md5 checksum in their metadata; refer to the matrix below [3]. If you have compatibility concerns for your workflow, please [submit an issue](https://github.com/broadinstitute/cromwell/issues). + +| Delocalization Strategy | Performance | crc32c | md5 | +|-------------------------|---------------|--------|-----| +| Classic | Baseline/slow | ✅ | ✅ | +| Parallel Composite | Fast | ✅ | ❌ | + +[0] Tested with Intel Ice Lake CPU platform, 16 vCPU, 32 GB RAM, 2500 GB SSD + +[1] [Throughput scales with vCPU count](https://cloud.google.com/compute/docs/disks/performance#n2_vms) with a plateau at 16 vCPUs. + +[2] [Throughput scales with disk size and type](https://cloud.google.com/compute/docs/disks/performance#throughput_limits_for_zonal) with at a plateau at 2.5 TB SSD. Worked example: 1200 MB/s ÷ 0.48 MB/s per GB = 2500 GB. + +[3] Cromwell itself uses crc32c hashes for call caching and is not affected + +## 86 Release Notes + +### GCP Batch +Cromwell now supports the GCP Batch backend for running workflows. See `Backend` in [ReadTheDocs](https://cromwell.readthedocs.io/en/stable/) for more information. + +### Workflow Completion Callback +Cromwell can be configured to send a POST request to a specified URL when a workflow completes. The request body includes the workflow ID, terminal state, +and (if applicable) final outputs or error message. See `WorkflowCallback` in [ReadTheDocs](https://cromwell.readthedocs.io/en/stable/) for more information. + +### Other Improvements +* Cromwell will now parallelize the downloads of DRS files that resolve to signed URLs. This significantly reduces the time localization takes in certain situations. +* WDL size engine function now works for HTTP files +* Improved Cromwell's handling of docker manifests. Additional logging information is emitted, and Cromwell will fall back to using OCI manifests if it encounters an error with a Docker Image Manifest V2. + +## 85 Release Notes + +### Migration of PKs to BIGINT + +The PK of below tables will be migrated from INT to BIGINT. Also, since `ROOT_WORKFLOW_ID` in `SUB_WORKFLOW_STORE_ENTRY` is a FK to `WORKFLOW_STORE_ENTRY_ID` in `WORKFLOW_STORE_ENTRY` +it is also being migrated from INT to BIGINT. +* DOCKER_HASH_STORE_ENTRY +* WORKFLOW_STORE_ENTRY +* SUB_WORKFLOW_STORE_ENTRY + +### Improvement to "retry with more memory" behavior + +Cromwell will now retry a task with more memory after it fails with return code 137, provided all +the other requirements for retrying with more memory are met. + +### DRS Improvements + +#### Support for invoking `CromwellDRSLocalizer` with manifest file + +`CromwellDRSLocalizer` can now handle multiple file localizations in a single invocation. Users can provide a +manifest file containing multiple (DRS id, local container path) pairs in CSV format, and they will be localized in +sequence, with the program exiting if any fail. +``` +java -jar /path/to/localizer.jar [options] -m /local/path/to/manifest/file.txt +``` + +The previous method of passing in a single DRS file and container destination using positional arguments is still +supported. + +#### Improvement to DRS localization in GCP papiv2beta backend + +All DRS inputs to a task are now localized in a single PAPI action, which should improve speed and resolve +failures observed when attempting to localize a large number of DRS files. + + +### Allow list for HTTP WDL resolution + +Administrators can now configure Cromwell with an allow list that limits the domains from which WDLs can be resolved and imported. +Default behavior is unchanged (Cromwell attempts to resolve WDL files from any URI). Example configuration: +``` +languages { + WDL { + http-allow-list { + enabled: true + allowed-http-hosts: [ + "my.wdl.repo.org", + "raw.githubusercontent.com" + ] + } + } +} +``` + +### CWL implementation removed + +This release removes the `cwl` top-level artifact. Some nonfunctional references may remain, and will be addressed over time. + +For more information, see the [Cromwell 79 release notes](https://github.com/broadinstitute/cromwell/releases/tag/79). + +### TES Improvments + +* Tes system errors are are now reported in Cromwell execution logs when the TES backend returns a task error. + +* Cromwell now attempts to translate `disks` attributes [written for GCP](https://cromwell.readthedocs.io/en/stable/RuntimeAttributes/#disks) into valid `disk` attributes for TES. For information on supported conversions, refer to the [TES documentation](https://cromwell.readthedocs.io/en/stable/backends/TES/). + +### Bug Fixes + +* Reference disks are only mounted if configured in the workflow options. + +* Recent docker images of Ubuntu use a new manifest format, ensure that these newer image versions can be pulled from Docker Registry without issue. + +* When converting ValueStore objects to strings for logging, we truncate long values to limit memory usage. + + +### Security Patching + +Updates to dependencies to fix security vulnerabilities. + +## 84 Release Notes + +### CromIAM enabled user checks + +For Cromwell instances utilizing the optional CromIAM identity and access management component, the following endpoints now verify that the calling user is enabled before forwarding the request. +* `/api/workflows/v1/backends` +* `/api/womtool/v1/describe` + +This change makes the above endpoints consistent with the existing behavior of all the other endpoints in the `/api/` path of CromIAM. + +## 83 Release Notes + +* Changes the type of several primary key columns in call caching tables from int to bigint. The database migration may be lengthy if your database contains a large amount of call caching data. + ## 82 Release Notes * Restored missing example configuration file @@ -27,7 +158,7 @@ The BCS backend and OSS filesystem (both of which support Alibaba Cloud) have be Cromwell 80 no longer supports the wes2cromwell project within the Cromwell repository. In the previous release, 3 Wes2Cromwell endpoints in the Cromwell project were implemented and documented in the Swagger API. Three new endpoints, -located within the wes2cromwell project, will also be moved, implemented, and documented within Cromwell. As a result of this, we can safely remove +located within the wes2cromwell project, will also be moved, implemented, and documented within Cromwell. As a result of this, we can safely remove and deprecate the wes2cromwell project from the repo. Previous endpoints: @@ -111,11 +242,11 @@ Previously: ### New 'requestedWorkflowId' API Option -Allows users to choose their own workflow IDs at workflow submission time. +Allows users to choose their own workflow IDs at workflow submission time. If supplied for single workflows, this value must be a JSON string containing a valid, and not already used, UUID. For batch submissions, this value must be a JSON array of valid UUIDs. -If not supplied, the behavior is as today: Cromwell will generate a random workflow ID for every workflow submitted. +If not supplied, the behavior is as today: Cromwell will generate a random workflow ID for every workflow submitted. ### Bug Fixes @@ -228,8 +359,8 @@ AccessURLs](https://ga4gh.github.io/data-repository-service-schemas/preview/rele ### No labels update for Archived workflows -If **- and ONLY if -** you have metadata archiving turned on, then for a workflow whose metadata has been archived by Cromwell -according to the lifecycle policy, Cromwell will no longer add new labels or update existing labels for this workflow +If **- and ONLY if -** you have metadata archiving turned on, then for a workflow whose metadata has been archived by Cromwell +according to the lifecycle policy, Cromwell will no longer add new labels or update existing labels for this workflow coming through PATCH `/labels` endpoint. ## 60 Release Notes @@ -241,12 +372,12 @@ containerized using [AdoptOpenJDK 11 HotSpot](https://adoptopenjdk.net/). ### Hybrid metadata storage ("carboniting") removed -Carboniting functionality has been removed from Cromwell. +Carboniting functionality has been removed from Cromwell. There will be no effect for customers who store metadata permanently in the relational database (most common), and there will also be no effect for customers who use the in-memory database. Breaking change only for customers who explicitly enabled `carbonite-metadata-service` in their configuration to split -metadata storage between a relational database and Google Cloud Storage. If you had previously enabled carboniting and +metadata storage between a relational database and Google Cloud Storage. If you had previously enabled carboniting and deletion, any workflows marked as `ArchivedAndPurged` in your database will no longer be accessible via the Cromwell metadata API. ## 59 Release Notes @@ -273,9 +404,9 @@ rather than through a level of indirection to a manifest file stored in GCS. Mor ### Retry with More Memory as workflow option The experimental memory retry feature gains per-workflow customization and includes breaking changes: -* The per-backend configuration key `.config.memory-retry.error-keys` has been removed and replaced +* The per-backend configuration key `.config.memory-retry.error-keys` has been removed and replaced with global key `system.memory-retry-error-keys` -* The per-backend configuration key `.config.memory-retry.multiplier` has been replaced with **workflow option** +* The per-backend configuration key `.config.memory-retry.multiplier` has been replaced with **workflow option** `memory_retry_multiplier` More details can be found [here](https://cromwell.readthedocs.io/en/develop/wf_options/Overview.md#retry-with-more-memory-multiplier). @@ -284,7 +415,7 @@ More details can be found [here](https://cromwell.readthedocs.io/en/develop/wf_o * Fixed a bug that caused Cromwell to mark workflows as failed after a single `500`, `503`, or `504` error from Google Cloud Storage. * Cromwell will now retry these errors as designed. - * The default retry count is `5` and may be customized with `system.io.number-of-attempts`. + * The default retry count is `5` and may be customized with `system.io.number-of-attempts`. ## 55 Release Notes @@ -294,7 +425,7 @@ Users with access to the new Mac hardware should review [important information p ### Bug Fixes -* Fixed a bug that prevented `read_json()` from working with arrays and primitives. The function now works as expected for all valid JSON data inputs. +* Fixed a bug that prevented `read_json()` from working with arrays and primitives. The function now works as expected for all valid JSON data inputs. More information on JSON Type to WDL Type conversion can be found [here](https://github.com/openwdl/wdl/blob/main/versions/1.0/SPEC.md#mixed-read_jsonstringfile). * Now retries HTTP 408 responses as well as HTTP 429 responses during DOS/DRS resolution requests. @@ -322,19 +453,19 @@ to the worker VM on the next attempt if the task is interrupted. More details [h ### Bug Fixes * Fixed a bug that prevented `write_json()` from working with arrays and primitives. The function now works as expected for `Boolean`, `String`, `Integer`, `Float`, - `Pair[_, _]`, `Object`, `Map[_, _]` and `Array[_]` (including array of objects) type inputs. More information on WDL Type to JSON Type + `Pair[_, _]`, `Object`, `Map[_, _]` and `Array[_]` (including array of objects) type inputs. More information on WDL Type to JSON Type conversion can be found [here](https://github.com/openwdl/wdl/blob/main/versions/1.0/SPEC.md#mixed-read_jsonstringfile). ### Spark backend support removal -Spark backend was not widely used and it was decided to remove it from the codebase in order to narrow the scope of Cromwell code. +Spark backend was not widely used and it was decided to remove it from the codebase in order to narrow the scope of Cromwell code. ### Improved DRS Localizer logging Error logging while localizing a DRS URI should now be more clear especially when there is a Requester Pays bucket involved. ### Per-backend hog factors -Cromwell now allows overriding system-level log factors on back-end level. First, Cromwell will try to use hog-factor +Cromwell now allows overriding system-level log factors on back-end level. First, Cromwell will try to use hog-factor defined in the backend config, and if it is not defined, it will default to using system-wide hog factor. ```conf backend { @@ -402,16 +533,16 @@ https://cromwell.readthedocs.io/en/stable/tutorials/Containers/#singularity). ### Google library upgrade [(#5565)](https://github.com/broadinstitute/cromwell/pull/5565) -All previous versions of Cromwell shipped with Google Cloud Storage (GCS) libraries that are now deprecated and will [stop working in August 2020](https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html). This release adopts updated libraries to ensure uninterrupted operation. The only user action required is upgrading Cromwell. +All previous versions of Cromwell shipped with Google Cloud Storage (GCS) libraries that are now deprecated and will [stop working in August 2020](https://developers.googleblog.com/2018/03/discontinuing-support-for-json-rpc-and.html). This release adopts updated libraries to ensure uninterrupted operation. The only user action required is upgrading Cromwell. ### Bug fixes * Fixed a bug that required Cromwell to be restarted in order to pick up DNS changes. * By default, the JVM caches DNS records with a TTL of infinity. - * Cromwell now configures its JVM with a 3-minute TTL. This value can be customized by setting `system.dns-cache-ttl`. + * Cromwell now configures its JVM with a 3-minute TTL. This value can be customized by setting `system.dns-cache-ttl`. * Clarified an error message that Cromwell emits when the compute backend terminates a job of its own volition (as opposed to termination in response to an abort request from Cromwell) * Previously, the error read `The job was aborted from outside Cromwell` - * The new error reads `The compute backend terminated the job. If this termination is unexpected, examine likely causes such as preemption, running out of disk or memory on the compute instance, or exceeding the backend's maximum job duration.` + * The new error reads `The compute backend terminated the job. If this termination is unexpected, examine likely causes such as preemption, running out of disk or memory on the compute instance, or exceeding the backend's maximum job duration.` ## 51 Release Notes @@ -422,8 +553,8 @@ https://cromwell.readthedocs.io/en/stable/Configuring/#call-caching) for details ### Bug fixes -* Fixed a bug where the `size(...)` function did not work correctly on files - from a shared filesystem if `size(...)` was called in the input section on a +* Fixed a bug where the `size(...)` function did not work correctly on files + from a shared filesystem if `size(...)` was called in the input section on a relative path. + Fixed a bug where the `use_relative_output_paths` option would not preserve intermediate folders. @@ -431,17 +562,17 @@ https://cromwell.readthedocs.io/en/stable/Configuring/#call-caching) for details #### Call caching blacklisting improvements -Cromwell previously supported blacklisting GCS buckets containing cache hits which could not be copied for permissions +Cromwell previously supported blacklisting GCS buckets containing cache hits which could not be copied for permissions reasons. Cromwell now adds support for blacklisting individual cache hits which could not be copied for any reason, as well as grouping blacklist caches according to a workflow option key. More information available in the [ -call caching documentation]( https://cromwell.readthedocs.io/en/stable/Configuring/#call-caching). +call caching documentation]( https://cromwell.readthedocs.io/en/stable/Configuring/#call-caching). #### new xxh64 and fingerprint strategies for call caching -Existing call cache strategies `path` and `path+modtime` don't work when using docker on shared filesystems +Existing call cache strategies `path` and `path+modtime` don't work when using docker on shared filesystems (SFS backend, i.e. not in cloud storage). The `file` (md5sum) strategy works, but uses a lot of resources. -Two faster strategies have been added for this use case: `xxh64` and -`fingerprint`. `xxh64` is a lightweight hashing algorithm, `fingerprint` is a strategy designed to be very +Two faster strategies have been added for this use case: `xxh64` and +`fingerprint`. `xxh64` is a lightweight hashing algorithm, `fingerprint` is a strategy designed to be very lightweight. Read more about it in the [call caching documentation]( https://cromwell.readthedocs.io/en/stable/Configuring/#call-caching). @@ -475,7 +606,7 @@ SELECT table_rows FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'cromwell' #### Execution Directory Layout (cache copies) -When an attempt to copy a cache result is made, you'll now see a `cacheCopy` directory in the call root directory. +When an attempt to copy a cache result is made, you'll now see a `cacheCopy` directory in the call root directory. This prevents them clashing with the files staged to the same directory for attempt 1 if the cache copy fails (see also: Bug Fixes). The directory layout used to be: @@ -512,7 +643,7 @@ but is now: #### Disable call-caching for tasks -It is now possible to indicate in a workflow that a task should not be call-cached. See details +It is now possible to indicate in a workflow that a task should not be call-cached. See details [here](https://cromwell.readthedocs.io/en/stable/optimizations/VolatileTasks). #### Delete Intermediate Outputs on PapiV2 @@ -525,13 +656,13 @@ for more information. #### Metadata Archival Support Cromwell 49 now offers the option to archive metadata to GCS and remove the equivalent metadata from relational -database storage. Please see -[the documentation](https://cromwell.readthedocs.io/en/stable/Configuring#hybrid-metadata-storage-classic-carbonite) for more details. +database storage. Please see +[the documentation](https://cromwell.readthedocs.io/en/stable/Configuring#hybrid-metadata-storage-classic-carbonite) for more details. #### Adding support for Google Cloud Life Sciences v2beta -Cromwell now supports running workflows using Google Cloud Life Sciences v2beta API in addition to Google Cloud Genomics v2alpha1. -More information about migration to the new API from v2alpha1 -[here](https://cromwell.readthedocs.io/en/stable/backends/Google#migration-from-google-cloud-genomics-v2alpha1-to-google-cloud-life-sciences-v2beta). +Cromwell now supports running workflows using Google Cloud Life Sciences v2beta API in addition to Google Cloud Genomics v2alpha1. +More information about migration to the new API from v2alpha1 +[here](https://cromwell.readthedocs.io/en/stable/backends/Google#migration-from-google-cloud-genomics-v2alpha1-to-google-cloud-life-sciences-v2beta). * **Note** Google Cloud Life Sciences is the new name for newer versions of Google Cloud Genomics. * **Note** Support for Google Cloud Genomics v2alpha1 will be removed in a future version of Cromwell. Advance notice will be provided. @@ -539,23 +670,23 @@ More information about migration to the new API from v2alpha1 #### Installation methods -Links to the conda package and docker container are now available in +Links to the conda package and docker container are now available in [the install documentation](https://cromwell.readthedocs.io/en/stable/Getting/). ### Bug Fixes -+ Fix a bug where zip files with directories could not be imported. - For example a zip with `a.wdl` and `b.wdl` could be imported but one with `sub_workflows/a.wdl` ++ Fix a bug where zip files with directories could not be imported. + For example a zip with `a.wdl` and `b.wdl` could be imported but one with `sub_workflows/a.wdl` and `imports/b.wdl` could not. + Fix a bug which sometimes allowed execution scripts copied by a failed cache-copy to be run instead - of the attempt-1 script for a live job execution. - + of the attempt-1 script for a live job execution. + ## 48 Release Notes ### Womtool Graph for WDL 1.0 -The `womtool graph` command now supports WDL 1.0 workflows. +The `womtool graph` command now supports WDL 1.0 workflows. * **Note:** Generated graphs - including in WDL draft 2 - may look slightly different than they did in version 47. ### Documentation @@ -567,7 +698,7 @@ The `womtool graph` command now supports WDL 1.0 workflows. ### Retry with more memory on Papiv2 [(#5180)](https://github.com/broadinstitute/cromwell/pull/5180) -Cromwell now allows user defined retries. With `memory-retry` config you can specify an array of strings which when encountered in the `stderr` +Cromwell now allows user defined retries. With `memory-retry` config you can specify an array of strings which when encountered in the `stderr` file by Cromwell, allows the task to be retried with multiplier factor mentioned in the config. More information [here](https://cromwell.readthedocs.io/en/stable/backends/Google/). ### GCS Parallel Composite Upload Support @@ -601,7 +732,7 @@ PAPI error code 14. Task was preempted for the 2nd time. ``` Cromwell 44 introduced special handling that detects both preemption indicators and re-runs the job consistent with the `preemptible` setting. -Cromwell 46 enhances this handling in response to user reports of possible continued issues. +Cromwell 46 enhances this handling in response to user reports of possible continued issues. ## 45 Release Notes @@ -619,7 +750,7 @@ More info [here](https://cromwell.readthedocs.io/en/stable/WOMtool/) ### BCS backend new Features support #### New docker registry -Alibaba Cloud Container Registry is now supported for the `docker` runtime attribute, and the previous `dockerTag` +Alibaba Cloud Container Registry is now supported for the `docker` runtime attribute, and the previous `dockerTag` runtime attribute continues to be available for Alibaba Cloud OSS Registry. #### Call caching Cromwell now supports Call caching when using the BCS backend. @@ -647,7 +778,7 @@ Cromwell's PAPI v2 backend will now handle this type of preemption. ### Virtual Private Cloud with Subnetworks -Cromwell now allows PAPIV2 jobs to run on a specific subnetwork inside a private network by adding the subnetwork key +Cromwell now allows PAPIV2 jobs to run on a specific subnetwork inside a private network by adding the subnetwork key `subnetwork-label-key` inside `virtual-private-cloud` in backend configuration. More info [here](https://cromwell.readthedocs.io/en/stable/backends/Google/). ### Call caching database refactoring @@ -655,7 +786,7 @@ Cromwell now allows PAPIV2 jobs to run on a specific subnetwork inside a private Cromwell's `CALL_CACHING_HASH_ENTRY` primary key has been refactored to use a `BIGINT` datatype in place of the previous `INT` datatype. Cromwell will not be usable during the time the Liquibase migration for this refactor is running. In the Google Cloud SQL with SSD environment this migration runs at a rate of approximately 100,000 `CALL_CACHING_HASH_ENTRY` -rows per second. In deployments with millions or billions of `CALL_CACHING_HASH_ENTRY` rows the migration may require +rows per second. In deployments with millions or billions of `CALL_CACHING_HASH_ENTRY` rows the migration may require a significant amount of downtime so please plan accordingly. The following SQL could be used to estimate the number of rows in this table: @@ -665,7 +796,7 @@ select max(CALL_CACHING_HASH_ENTRY_ID) from CALL_CACHING_HASH_ENTRY ### Stackdriver Instrumentation -Cromwell now supports sending metrics to [Google's Stackdriver API](https://cloud.google.com/monitoring/api/v3/). +Cromwell now supports sending metrics to [Google's Stackdriver API](https://cloud.google.com/monitoring/api/v3/). Learn more on how to configure [here](https://cromwell.readthedocs.io/en/stable/developers/Instrumentation/). ### BigQuery in PAPI @@ -682,7 +813,7 @@ which now has been updated to `services.Instrumentation.config`. More info on it #### cached-copy -A new experimental feature, the `cached-copy` localization strategy is available for the shared filesystem. +A new experimental feature, the `cached-copy` localization strategy is available for the shared filesystem. More information can be found in the [documentation on localization](https://cromwell.readthedocs.io/en/stable/backends/HPC). #### Yaml node limits @@ -716,7 +847,7 @@ Specifically, the new `validWorkflow` key indicates whether the workflow file is ### Configuration Changes * Virtual private networks can now be configured. See the section below for details. - + #### Batch Request Timeouts The timeout on Cromwell's requests to PAPIv2 can now be configured. See the sample PAPIv2.conf for more documentation: @@ -725,7 +856,7 @@ The timeout on Cromwell's requests to PAPIv2 can now be configured. See the samp backend { providers { PAPIv2 { - config { + config { batch-requests { timeouts { read = 10 seconds @@ -802,14 +933,14 @@ This field is now accepted within WDL files as well as within the configuration #### Logging long running jobs -All backends can now emit slow job warnings after a configurable time running. +All backends can now emit slow job warnings after a configurable time running. NB This example shows how to configure this setting for the PAPIv2 backend: ```conf # Emit a warning if jobs last longer than this amount of time. This might indicate that something got stuck. backend { providers { PAPIv2 { - config { + config { slow-job-warning-time: 24 hours } } @@ -824,7 +955,7 @@ backend { * The `gpuType` attribute is no longer validated against a whitelist at workflow submission time. Instead, validation now happens at runtime. This allows any valid accelerator to be used. * The `nvidiaDriverVersion` attribute is now available in WDL `runtime` sections. The default continues to be `390.46` which applies if and only if GPUs are being used. * A default `gpuType` ("nvidia-tesla-k80") will now be applied if `gpuCount` is specified but `gpuType` is not. -* Similarly, a default `gpuCount` (1) will be applied if `gpuType` is specified but `cpuCount` is not. +* Similarly, a default `gpuCount` (1) will be applied if `gpuType` is specified but `cpuCount` is not. ### Bug fixes @@ -865,7 +996,7 @@ services { class = "cromwell.services.healthmonitor.impl.standard.StandardHealthMonitorServiceActor" } } -``` +``` With this one: ``` services { @@ -876,7 +1007,7 @@ services { } } } -``` +``` ###### From `WorkbenchHealthMonitorServiceActor`: Replace this stanza: ``` @@ -893,7 +1024,7 @@ services { } } } -``` +``` With this one: ``` services { @@ -909,12 +1040,12 @@ services { } } } -``` +``` ### Workflow options changes -A new workflow option is added. If the `final_workflow_outputs_dir` is set -`use_relative_output_paths` can be used. When set to `true` this will copy -all the outputs relative to their execution directory. +A new workflow option is added. If the `final_workflow_outputs_dir` is set +`use_relative_output_paths` can be used. When set to `true` this will copy +all the outputs relative to their execution directory. my_final_workflow_outputs_dir/~~MyWorkflow/af76876d8-6e8768fa/call-MyTask/execution/~~output_of_interest. More information can be found in [the workflow options documentation](https://cromwell.readthedocs.io/en/stable/wf_options/Overview/#output-copying). @@ -944,10 +1075,10 @@ shutdown. ### Bug fixes -#### Format fix for `write_map()` +#### Format fix for `write_map()` Fixed an issue that caused the `write_map()` function in Cromwell's WDL 1.0 implementation to produce output in the wrong format. Specifically, the output's rows and columns were swapped. WDL draft-2 was not affected. - + Incorrect `write_map()` output in Cromwell 38 and earlier: ``` key1 key2 key3 @@ -987,7 +1118,7 @@ available as `StandardHealthMonitorServiceActor`. ### Docker - Adds support for retrieving docker digests of asia.gcr.io images -- Adds configuration settings for docker digest lookups. See the `docker` section of the `reference.conf` for more information +- Adds configuration settings for docker digest lookups. See the `docker` section of the `reference.conf` for more information - Attempt to automatically adjust the boot disk size on the Google Cloud Backend (version 2) if the size of the image is greater than the default disk size or the required disk size in the runtime attributes. Only works for registries that support the version 2 of the manifest schema (https://docs.docker.com/registry/spec/manifest-v2-2/) At this date (12/09/18) this includes GCR and Dockerhub. @@ -1032,28 +1163,28 @@ Support `InputResourceRequirement` hint In cases where its not obvious why jobs are queued in Cromwell, you can enable logging for the Job Execution Token Dispenser, using the `system.hog-safety.token-log-interval-seconds` configuration value. -The default, `0`, means that no logging will occur. +The default, `0`, means that no logging will occur. #### HTTP Filesystem -- The HTTP filesystem is now enabled for engine use by default. To continue without an HTTP filesystem, you can add the +- The HTTP filesystem is now enabled for engine use by default. To continue without an HTTP filesystem, you can add the following content into the appropriate stanza of your configuration file: ``` engine { filesystems { - http { - enabled: false + http { + enabled: false } } } -``` +``` - When the value `exit-code-timeout-seconds` is set, `check-alive` command is now only called once every timeout interval instead of each poll. ### Beta preview of new Womtool `/describe` endpoint This new endpoint brings the functionality of Womtool to the world of web services. Submit workflows for validation and receive a JSON description in response. -The endpoint is still undergoing heavy development and should not be used in production. The final version will ship in a future release of Cromwell; watch this space. +The endpoint is still undergoing heavy development and should not be used in production. The final version will ship in a future release of Cromwell; watch this space. ### Bug fixes @@ -1093,7 +1224,7 @@ Details [here](https://cromwell.readthedocs.io/en/develop/backends/HPC/#exit-cod Coalesce metadata requests to eliminate expensive and redundant queries and metadata construction. -#### Eliminate redundant SFS logging and metadata +#### Eliminate redundant SFS logging and metadata Eliminate superfluous logging and metadata publishing in the shared filesystem backend on poll intervals where there was not a state change. @@ -1106,7 +1237,7 @@ Previously US-EAST-1 was hardcoded in places. ### Submit workflow using URL Cromwell now allows for a user to submit the URL pointing to workflow file to run a workflow. -More details on how to use it in: +More details on how to use it in: - `Server` mode can be found [here](https://cromwell.readthedocs.io/en/develop/api/RESTAPI/). - `Run` mode can be found [here](https://cromwell.readthedocs.io/en/develop/CommandLine/#run). @@ -1132,7 +1263,7 @@ Cromwell now offers the ability to cache file hashes on a root workflow level ba ### Extra configuration options -The value `dockerRoot` can now be set in a backend configuration. +The value `dockerRoot` can now be set in a backend configuration. This will set the execution folder in the container (default: `/cromwell-executions`). ### Bug Fixes @@ -1238,7 +1369,7 @@ Cromwell now supports retrying failed tasks up to a specified count by declaring * Cromwell now publishes the labels as soon as the workflow is submitted (whether started or on hold). If the labels are invalid, the workflow will not be submitted and request will fail. ### Scala 2.11 Removed -From version 32 onwards we will no longer be publishing build artifacts compatible with Scala 2.11. +From version 32 onwards we will no longer be publishing build artifacts compatible with Scala 2.11. * If you don't import the classes into your own scala project then this should have no impact on you. * If you **are** importing the classes into your own scala project, make sure you are using Scala 2.12. @@ -1330,12 +1461,12 @@ Workflow metadata for jobs run on a Google Pipelines API backend will report the ## 31 Release Notes -* **Cromwell server** -The Cromwell server source code is now located under `server/src`. `sbt assembly` will build the runnable Cromwell JAR in +* **Cromwell server** +The Cromwell server source code is now located under `server/src`. `sbt assembly` will build the runnable Cromwell JAR in `server/target/scala-2.12/` with a name like `cromwell-.jar`. * **Robustness** - + The rate at which jobs are being started can now be controlled using the `system.job-rate-control` configuration stanza. + + The rate at which jobs are being started can now be controlled using the `system.job-rate-control` configuration stanza. + A load controller service has been added to allow Cromwell to self-monitor and adjust its load accordingly. The load controller is currently a simple on/off switch controlling the job start rate. It gathers metrics from different parts of the system to inform its decision to stop the creation of jobs. @@ -1343,7 +1474,7 @@ You can find relevant configuration in the `services.LoadController` section of as well as in the `load-control` section in `reference.conf`. The load level of the monitored sub-systems are instrumented and can be found under the `cromwell.load` statsD path. + The statsD metrics have been re-shuffled a bit. If you had a dashboard you might find that you need to update it. -Changes include: +Changes include: + Removed artificially inserted "count" and "timing" the path + Added a `load` section + Metrics were prefixed twice with `cromwell` (`cromwell.cromwell.my_metric`), now they're only prefixed once @@ -1352,8 +1483,8 @@ Changes include: * Added a configuration option under `docker.hash-lookup.enabled` to disable docker hash lookup. Disabling it will also disable call caching for jobs with floating docker tags. - -* **API** + +* **API** + Updated the `/query` response to include the total number of query results returned. See [here](http://cromwell.readthedocs.io/en/develop/api/RESTAPI/#workflowqueryresponse) for more information. ## 30.1 Release Notes @@ -1368,14 +1499,14 @@ Changes include: ### Other changes -* **New Cromwell documentation** +* **New Cromwell documentation** Our documentation has moved from our [README](https://github.com/broadinstitute/cromwell/blob/29_hotfix/README.md) to a new website: [Cromwell Documentation](http://cromwell.readthedocs.io/en/develop/). There are new [Tutorials](http://cromwell.readthedocs.io/en/develop/tutorials/FiveMinuteIntro/) and much of the documentation has been re-written. The source files are in the [/docs](https://github.com/broadinstitute/cromwell/tree/develop/docs) directory. -* **API** +* **API** + Cromwell now supports input files in the yaml format (JSON format is still supported). + Added a [GET version for the `labels` endpoint](http://cromwell.readthedocs.io/en/develop/api/RESTAPI/#retrieves-the-current-labels-for-a-workflow) which will return current labels for a workflow. -* **Database** +* **Database** You have the option of storing the metadata in a separate SQL database than the database containing the internal engine data. When switching connection information for an existing database containing historical data, the tables should be manually replicated from one database instance to another using the tools appropriate for your specific @@ -1384,48 +1515,48 @@ and likely to change in the future. See the [Database Documentation](https://cro [cromwell.examples.conf](https://www.github.com/broadinstitute/cromwell/tree/develop/cromwell.example.backends/cromwell.examples.conf) for more information. -* **StatsD** +* **StatsD** Added initial support for StatsD instrumentation. See the [Instrumentation Documentation](https://cromwell.readthedocs.io/en/develop/Instrumentation) for details on how to use it. -* **User Service Account auth mode for Google** +* **User Service Account auth mode for Google** Added a new authentication mode for [Google Cloud Platform](https://cromwell.readthedocs.io/en/develop/backends/Google) which will allow a user to supply the JSON key file in their workflow options to allow for per-workflow authentication via service account. This is analogous to the previously existing refresh token authentication scheme. As with the refresh token scheme it is encouraged that the **user_service_account_json** workflow option field is added to the **encrypted-fields** list in the configuration. -* **Bugfixes** +* **Bugfixes** Abort of Dockerized tasks on the Local backend should now work as expected. Cromwell uses `docker kill` to kill the Docker container. ## 29 Release Notes ### Breaking Changes -* **Command line** +* **Command line** In preparation for supporting CWL scripts (yes, you read that right!), we have extensively revised the Command Line in Cromwell 29. For more details about the usage changes please see the [README](https://github.com/broadinstitute/cromwell#command-line-usage). And stay tuned to the [WDL/Cromwell blog](https://software.broadinstitute.org/wdl/blog) over the next couple of months for more news about CWL. -* **Request timeouts** +* **Request timeouts** Cromwell now returns more specific `503 Service Unavailable` error codes on request timeouts, rather than the more generic `500 Internal Server Error`. The response for a request timeout will now be plain text, rather than a JSON format. -* **Metadata endpoint** +* **Metadata endpoint** The response from the metadata endpoint can be quite large depending on your workflow. You can now opt-in to have Cromwell gzip your metadata file, in order to reduce file size, by sending the `Accept-Encoding: gzip` header. The default behavior now does not gzip encode responses. -* **Engine endpoints** +* **Engine endpoints** Previously the engine endpoints were available under `/api/engine` but now the endpoints are under `/engine` so they don't require authentication. Workflow endpoints are still available under `/api/workflows`. We also deprecated the setting `api.routeUnwrapped` as a part of this internal consistency effort. -* **Call caching diff** +* **Call caching diff** We updated the response format of the [callcaching/diff](https://github.com/broadinstitute/cromwell#get-apiworkflowsversioncallcachingdiff) endpoint. ### Other changes -* **Cromwell server** +* **Cromwell server** When running in server mode, Cromwell now attempts to gracefully shutdown after receiving a `SIGINT` (`Ctrl-C`) or `SIGTERM` (`kill`) signal. This means that Cromwell waits for all pending database writes before exiting, as long as you include `application.conf` at the top of your config file. You can find detailed information about how to configure this feature in the [Cromwell Wiki](https://github.com/broadinstitute/cromwell/wiki/DevZone#graceful-server-shutdown). -* **Concurrent jobs** +* **Concurrent jobs** You can now limit the number of concurrent jobs for any backend. Previously this was only possible in some backend implementations. Please see the [README](https://github.com/broadinstitute/cromwell#backend-job-limits) for details. ### WDL -* **Optional WDL variables** +* **Optional WDL variables** Empty optional WDL values are now rendered as the `null` JSON value instead of the JSON string `"null"` in the metadata and output endpoints. You do not need to migrate previous workflows. Workflows run on Cromwell 28 and prior will still render empty values as `"null"`. -* **Empty WDL variables** +* **Empty WDL variables** Cromwell now accepts `null` JSON values in the input file and coerces them as an empty WDL value. WDL variables must be declared optional in order to be supplied with a `null` JSON value. input.json @@ -1474,8 +1605,8 @@ task writer { #### `ContinueWhilePossible` A workflow utilizing the WorkflowFailureMode Workflow Option `ContinueWhilePossible` will now successfully reach a terminal state once all runnable jobs have completed. -#### `FailOnStderr` -When `FailOnStderr` is set to false, Cromwell no longer checks for the existence of a stderr file for that task. +#### `FailOnStderr` +When `FailOnStderr` is set to false, Cromwell no longer checks for the existence of a stderr file for that task. ### WDL Functions @@ -1491,7 +1622,7 @@ task foo { command { ... } runtime { docker: "..." - memory: ceil(size(in_file)) * 4 + memory: ceil(size(in_file)) * 4 } } ``` @@ -1506,7 +1637,7 @@ If the hashes fail to be calculated, the reason is indicated in a `hashFailures` See the [README](https://github.com/broadinstitute/cromwell#get-apiworkflowsversionidmetadata) for an example metadata response. -* New endpoint returning the hash differential for 2 calls. +* New endpoint returning the hash differential for 2 calls. `GET /api/workflows/:version/callcaching/diff` @@ -1535,7 +1666,7 @@ A second value is allowed, `reference`, that will instead point to the original filesystems { gcs { auth = "application-default" - + caching { duplication-strategy = "reference" } @@ -1563,7 +1694,7 @@ than the previous default. The migration will only be executed on MySQL. Other databases will lose their previous cached jobs. In order to run properly on MySQL, **the following flag needs to be adjusted**: https://dev.mysql.com/doc/refman/5.5/en/server-system-variables.html#sysvar_group_concat_max_len The following query will give you a minimum to set the group_concat_max_len value to: - + ```sql SELECT MAX(aggregated) as group_concat_max_len FROM ( @@ -1574,15 +1705,15 @@ SELECT MAX(aggregated) as group_concat_max_len FROM ``` Here is the SQL command to run to set the group_concat_max_len flag to the proper value: - + ```sql SET GLOBAL group_concat_max_len = value ``` - + Where `value` is replaced with the value you want to set it to. - + Note that the migration will fail if the flag is not set properly. - + ### Breaking Changes * The update to Slick 3.2 requires a database stanza to @@ -1630,7 +1761,7 @@ run the call: ### Docker -* The Docker section of the configuration has been slightly reworked +* The Docker section of the configuration has been slightly reworked An option to specify how a Docker hash should be looked up has been added. Two methods are available. "local" will try to look for the image on the machine where cromwell is running. If it can't be found, Cromwell will try to `pull` the image and use the hash from the retrieved image. "remote" will try to look up the image hash directly on the remote repository where the image is located (Docker Hub and GCR are supported) @@ -1694,14 +1825,14 @@ failures: [{ * Added a configuration option under `system.io` to throttle the number of I/O queries that Cromwell makes, as well as configure retry parameters. This is mostly useful for the JES backend and should be updated to match the GCS quota available for the project. - + ``` system.io { # Global Throttling - This is mostly useful for GCS and can be adjusted to match # the quota availble on the GCS API number-of-requests = 100000 per = 100 seconds - + # Number of times an I/O operation should be attempted before giving up and failing it. number-of-attempts = 5 } @@ -1725,7 +1856,7 @@ system.io { ### Config Changes -* Added a field `insert-batch-size` to the `database` stanza which defines how many values from a batch insert will be processed at a time. This value defaults to 2000. +* Added a field `insert-batch-size` to the `database` stanza which defines how many values from a batch insert will be processed at a time. This value defaults to 2000. * Moved the config value `services.MetadataService.metadata-summary-refresh-interval` to `services.MetadataService.config.metadata-summary-refresh-interval` * Added ability to override the default zone(s) used by JES via the config structure by setting `genomics.default-zones` in the JES configuration * The cromwell server TCP binding timeout is now configurable via the config key `webservice.binding-timeout`, defaulted @@ -1748,15 +1879,15 @@ system.io { * In some cases the SFS backend, used for Local, SGE, etc., coerced `WdlFile` to `WdlString` by using `.toUri`. This resulted in strings prepended with `file:///path/to/file`. Now absolute file paths will not contain the uri scheme. * Launch jobs on servers that support the GA4GH Task Execution Schema using the TES backend. -* **Call caching: Cromwell will no longer try to use the cache for WDL tasks that contain a floating docker tag.** +* **Call caching: Cromwell will no longer try to use the cache for WDL tasks that contain a floating docker tag.** Call caching will still behave the same for tasks having a docker image with a specific hash. - See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. + See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. * Added docker hash lookup. Cromwell will try to lookup the hash for a docker image with a floating tag, and use that hash when executing the job. This will be reflected in the metadata where the docker runtime attribute will contains the hash that was used. If Cromwell is unable to lookup the docker hash, the job will be run with the original user defined floating tag. Cromwell is currently able to lookup public and private docker hashes for images on Docker Hub and Google Container Engine for job running on the JES backend. For other backends, cromwell is able to lookup public docker hashes for Docker Hub and Google Container Engine. - See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. + See https://github.com/broadinstitute/cromwell#call-caching-docker-tags for more details. ### Database schema changes * Added CUSTOM_LABELS as a field of WORKFLOW_STORE_ENTRY, to store workflow store entries. @@ -1775,7 +1906,7 @@ resulted in strings prepended with `file:///path/to/file`. Now absolute file pat * Timing diagrams and metadata now receive more fine grained workflow states between submission and Running. * Support for the Pair WDL type (e.g. `Pair[Int, File] floo = (3, "gs://blar/blaz/qlux.txt")`) * Added support for new WDL functions: - * `zip: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - align items in the two arrays by index and return them as WDL pairs + * `zip: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - align items in the two arrays by index and return them as WDL pairs * `cross: (Array[X], Array[Y]) => Array[Pair[X, Y]]` - create every possible pair from the two input arrays and return them all as WDL pairs * `transpose: (Array[Array[X]]) => Array[Array[X]]` compute the matrix transpose for a 2D array. Assumes each inner array has the same length. * By default, `system.abort-jobs-on-terminate` is false when running `java -jar cromwell.jar server`, and true when running `java -jar cromwell.jar run `. @@ -1791,9 +1922,9 @@ resulted in strings prepended with `file:///path/to/file`. Now absolute file pat * Fix bugs related to the behavior of Cromwell in Single Workflow Runner Mode. Cromwell will now exit once a workflow completes in Single Workflow Runner Mode. Additionally, when restarting Cromwell in Single Workflow Runner Mode, Cromwell will no longer restart incomplete workflows from a previous session. ### Annex A - Workflow outputs - + The WDL specification has changed regarding [workflow outputs](https://github.com/openwdl/wdl/blob/master/versions/draft-2/SPEC.md#outputs) to accommodate sub workflows. -This change is backward compatible in terms of runnable WDLs (WDL files using the deprecated workflow outputs syntax will still run the same). +This change is backward compatible in terms of runnable WDLs (WDL files using the deprecated workflow outputs syntax will still run the same). The only visible change lies in the metadata (as well as the console output in single workflow mode, when workflow outputs are printed out at the end of a successful workflow). TL;DR Unless you are parsing or manipulating the "key" by which workflow outputs are referenced in the metadata (and/or the console output for single workflow mode), you can skip the following explanation. @@ -1846,7 +1977,7 @@ task t { The new syntax allows for type checking of the outputs as well as expressions. It also allows for explicitly naming to the outputs. The old syntax doesn't give the ability to name workflow outputs. For consistency reasons, Cromwell will generate a "new syntax" workflow output for each task output, and name them. -Their name will be generated using their FQN, which would give +Their name will be generated using their FQN, which would give ``` output { @@ -1854,8 +1985,8 @@ output { String w.t.out2 = t.out2 } ``` - -However as the FQN separator is `.`, the name itself cannot contain any `.`. + +However as the FQN separator is `.`, the name itself cannot contain any `.`. For that reason, `.` are replaced with `_` : *Old syntax expanded to new syntax* @@ -1866,16 +1997,16 @@ output { } ``` -The consequence is that the workflow outputs section of the metadata for `old_syntax` would previously look like - +The consequence is that the workflow outputs section of the metadata for `old_syntax` would previously look like + ``` outputs { "w.t.out1": "hello", "w.t.out2": "hello" } ``` - -but it will now look like + +but it will now look like ``` outputs { diff --git a/CODEOWNERS b/CODEOWNERS new file mode 100644 index 00000000000..34ece8d7792 --- /dev/null +++ b/CODEOWNERS @@ -0,0 +1,5 @@ +# These owners will be the default owners for everything in +# the repo. Unless a later match takes precedence, +# @broadinstitute/dsp-batch will be requested for +# review when someone opens a pull request. +* @broadinstitute/dsp-batch diff --git a/CromIAM/src/main/scala/cromiam/auth/Collection.scala b/CromIAM/src/main/scala/cromiam/auth/Collection.scala index 84e72194ab2..47785ae2005 100644 --- a/CromIAM/src/main/scala/cromiam/auth/Collection.scala +++ b/CromIAM/src/main/scala/cromiam/auth/Collection.scala @@ -9,6 +9,7 @@ import scala.util.{Success, Try} final case class Collection(name: String) extends AnyVal object Collection { + /** * Parses a raw JSON string to make sure it fits the standard pattern (see below) for labels, * performs some CromIAM-specific checking to ensure the user isn't attempting to manipulate the @@ -19,13 +20,14 @@ object Collection { */ def validateLabels(labelsJson: Option[String]): Directive1[Option[Map[String, JsValue]]] = { - val labels = labelsJson map { l => - Try(l.parseJson) match { - case Success(JsObject(json)) if json.keySet.contains(CollectionLabelName) => throw new LabelContainsCollectionException - case Success(JsObject(json)) => json - case _ => throw InvalidLabelsException(l) - } + val labels = labelsJson map { l => + Try(l.parseJson) match { + case Success(JsObject(json)) if json.keySet.contains(CollectionLabelName) => + throw new LabelContainsCollectionException + case Success(JsObject(json)) => json + case _ => throw InvalidLabelsException(l) } + } provide(labels) } @@ -34,15 +36,16 @@ object Collection { val LabelsKey = "labels" // LabelContainsCollectionException is a class because of ScalaTest, some of the constructs don't play well w/ case objects - final class LabelContainsCollectionException extends Exception(s"Submitted labels contain the key $CollectionLabelName, which is not allowed\n") - final case class InvalidLabelsException(labels: String) extends Exception(s"Labels must be a valid JSON object, received: $labels\n") + final class LabelContainsCollectionException + extends Exception(s"Submitted labels contain the key $CollectionLabelName, which is not allowed\n") + final case class InvalidLabelsException(labels: String) + extends Exception(s"Labels must be a valid JSON object, received: $labels\n") /** * Returns the default collection for a user. */ - def forUser(user: User): Collection = { + def forUser(user: User): Collection = Collection(user.userId.value) - } implicit val collectionJsonReader = new JsonReader[Collection] { import spray.json.DefaultJsonProtocol._ diff --git a/CromIAM/src/main/scala/cromiam/auth/User.scala b/CromIAM/src/main/scala/cromiam/auth/User.scala index d123f8fa2f7..fec64ebc5f9 100644 --- a/CromIAM/src/main/scala/cromiam/auth/User.scala +++ b/CromIAM/src/main/scala/cromiam/auth/User.scala @@ -7,4 +7,3 @@ import org.broadinstitute.dsde.workbench.model.WorkbenchUserId * Wraps the concept of an authenticated workbench user including their numeric ID as well as their bearer token */ final case class User(userId: WorkbenchUserId, authorization: Authorization) - diff --git a/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala b/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala index a95b8df89b5..4a419b62255 100644 --- a/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala +++ b/CromIAM/src/main/scala/cromiam/cromwell/CromwellClient.scala @@ -25,10 +25,16 @@ import scala.concurrent.{ExecutionContextExecutor, Future} * * FIXME: Look for ways to synch this up with the mothership */ -class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingAdapter, serviceRegistryActorRef: ActorRef)(implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends SprayJsonSupport with DefaultJsonProtocol with StatusCheckedSubsystem with CromIamInstrumentation{ +class CromwellClient(scheme: String, + interface: String, + port: Int, + log: LoggingAdapter, + serviceRegistryActorRef: ActorRef +)(implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) + extends SprayJsonSupport + with DefaultJsonProtocol + with StatusCheckedSubsystem + with CromIamInstrumentation { val cromwellUrl = new URL(s"$scheme://$interface:$port") val cromwellApiVersion = "v1" @@ -41,21 +47,23 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA def collectionForWorkflow(workflowId: String, user: User, - cromIamRequest: HttpRequest): FailureResponseOrT[Collection] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Collection] = { import CromwellClient.EnhancedWorkflowLabels log.info("Requesting collection for " + workflowId + " for user " + user.userId + " from metadata") // Look up in Cromwell what the collection is for this workflow. If it doesn't exist, fail the Future - val cromwellApiLabelFunc = () => cromwellApiClient.labels(WorkflowId.fromString(workflowId), headers = List(user.authorization)) flatMap { - _.caasCollection match { - case Some(c) => FailureResponseOrT.pure[IO, HttpResponse](c) - case None => - val exception = new IllegalArgumentException(s"Workflow $workflowId has no associated collection") - val failure = IO.raiseError[Collection](exception) - FailureResponseOrT.right[HttpResponse](failure) + val cromwellApiLabelFunc = () => + cromwellApiClient.labels(WorkflowId.fromString(workflowId), headers = List(user.authorization)) flatMap { + _.caasCollection match { + case Some(c) => FailureResponseOrT.pure[IO, HttpResponse](c) + case None => + val exception = new IllegalArgumentException(s"Workflow $workflowId has no associated collection") + val failure = IO.raiseError[Collection](exception) + FailureResponseOrT.right[HttpResponse](failure) + } } - } instrumentRequest(cromwellApiLabelFunc, cromIamRequest, wfCollectionPrefix) } @@ -63,13 +71,14 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA def forwardToCromwell(httpRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { val future = { // See CromwellClient's companion object for info on these header modifications - val headers = httpRequest.headers.filterNot(header => header.name == TimeoutAccessHeader || header.name == HostHeader) + val headers = + httpRequest.headers.filterNot(header => header.name == TimeoutAccessHeader || header.name == HostHeader) val cromwellRequest = httpRequest .copy(uri = httpRequest.uri.withAuthority(interface, port).withScheme(scheme)) .withHeaders(headers) Http().singleRequest(cromwellRequest) - } recoverWith { - case e => Future.failed(CromwellConnectionFailure(e)) + } recoverWith { case e => + Future.failed(CromwellConnectionFailure(e)) } future.asFailureResponseOrT } @@ -86,7 +95,7 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA use the current workflow id. This is all called from inside the context of a Future, so exceptions will be properly caught. - */ + */ metadata.value.parseJson.asJsObject.fields.get("rootWorkflowId").map(_.convertTo[String]).getOrElse(workflowId) } @@ -96,11 +105,13 @@ class CromwellClient(scheme: String, interface: String, port: Int, log: LoggingA Grab the metadata from Cromwell filtered down to the rootWorkflowId. Then transform the response to get just the root workflow ID itself */ - val cromwellApiMetadataFunc = () => cromwellApiClient.metadata( - WorkflowId.fromString(workflowId), - args = Option(Map("includeKey" -> List("rootWorkflowId"))), - headers = List(user.authorization)).map(metadataToRootWorkflowId - ) + val cromwellApiMetadataFunc = () => + cromwellApiClient + .metadata(WorkflowId.fromString(workflowId), + args = Option(Map("includeKey" -> List("rootWorkflowId"))), + headers = List(user.authorization) + ) + .map(metadataToRootWorkflowId) instrumentRequest(cromwellApiMetadataFunc, cromIamRequest, rootWfIdPrefix) } @@ -120,14 +131,14 @@ object CromwellClient { // See: https://broadworkbench.atlassian.net/browse/DDO-2190 val HostHeader = "Host" - final case class CromwellConnectionFailure(f: Throwable) extends Exception(s"Unable to connect to Cromwell (${f.getMessage})", f) + final case class CromwellConnectionFailure(f: Throwable) + extends Exception(s"Unable to connect to Cromwell (${f.getMessage})", f) implicit class EnhancedWorkflowLabels(val wl: WorkflowLabels) extends AnyVal { - import Collection.{CollectionLabelName, collectionJsonReader} + import Collection.{collectionJsonReader, CollectionLabelName} - def caasCollection: Option[Collection] = { + def caasCollection: Option[Collection] = wl.labels.fields.get(CollectionLabelName).map(_.convertTo[Collection]) - } } } diff --git a/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala b/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala index 63f48073146..c79e0b6e14c 100644 --- a/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala +++ b/CromIAM/src/main/scala/cromiam/instrumentation/CromIamInstrumentation.scala @@ -20,16 +20,18 @@ trait CromIamInstrumentation extends CromwellInstrumentation { val samPrefix: NonEmptyList[String] = NonEmptyList.one("sam") val getWhitelistPrefix = NonEmptyList.one("get-whitelist") + val getUserEnabledPrefix = NonEmptyList.one("get-user-enabled") val userCollectionPrefix = NonEmptyList.one("user-collection") val authCollectionPrefix = NonEmptyList.one("auth-collection") val registerCollectionPrefix = NonEmptyList.one("register-collection") val rootWfIdPrefix = NonEmptyList.one("root-workflow-id") val wfCollectionPrefix = NonEmptyList.one("workflow-collection") - def convertRequestToPath(httpRequest: HttpRequest): NonEmptyList[String] = NonEmptyList.of( // Returns the path of the URI only, without query parameters (e.g: api/engine/workflows/metadata) - httpRequest.uri.path.toString().stripPrefix("/") + httpRequest.uri.path + .toString() + .stripPrefix("/") // Replace UUIDs with [id] to keep paths same regardless of the workflow .replaceAll(CromIamInstrumentation.UUIDRegex, "[id]"), // Name of the method (e.g: GET) @@ -42,15 +44,19 @@ trait CromIamInstrumentation extends CromwellInstrumentation { def makePathFromRequestAndResponse(httpRequest: HttpRequest, httpResponse: HttpResponse): InstrumentationPath = convertRequestToPath(httpRequest).concatNel(NonEmptyList.of(httpResponse.status.intValue.toString)) - def sendTimingApi(statsDPath: InstrumentationPath, timing: FiniteDuration, prefixToStatsd: NonEmptyList[String]): Unit = { + def sendTimingApi(statsDPath: InstrumentationPath, + timing: FiniteDuration, + prefixToStatsd: NonEmptyList[String] + ): Unit = sendTiming(prefixToStatsd.concatNel(statsDPath), timing, CromIamPrefix) - } - def instrumentationPrefixForSam(methodPrefix: NonEmptyList[String]): NonEmptyList[String] = samPrefix.concatNel(methodPrefix) + def instrumentationPrefixForSam(methodPrefix: NonEmptyList[String]): NonEmptyList[String] = + samPrefix.concatNel(methodPrefix) def instrumentRequest[A](func: () => FailureResponseOrT[A], httpRequest: HttpRequest, - prefix: NonEmptyList[String]): FailureResponseOrT[A] = { + prefix: NonEmptyList[String] + ): FailureResponseOrT[A] = { def now(): Deadline = Deadline.now val startTimestamp = now() diff --git a/CromIAM/src/main/scala/cromiam/sam/SamClient.scala b/CromIAM/src/main/scala/cromiam/sam/SamClient.scala index f289251a2fb..8fa0cc8fd87 100644 --- a/CromIAM/src/main/scala/cromiam/sam/SamClient.scala +++ b/CromIAM/src/main/scala/cromiam/sam/SamClient.scala @@ -18,6 +18,7 @@ import cromiam.sam.SamResourceJsonSupport._ import cromiam.server.status.StatusCheckedSubsystem import cromwell.api.model._ import mouse.boolean._ +import spray.json.RootJsonFormat import scala.concurrent.ExecutionContextExecutor @@ -32,20 +33,21 @@ class SamClient(scheme: String, port: Int, checkSubmitWhitelist: Boolean, log: LoggingAdapter, - serviceRegistryActorRef: ActorRef) - (implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) extends StatusCheckedSubsystem with CromIamInstrumentation { + serviceRegistryActorRef: ActorRef +)(implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) + extends StatusCheckedSubsystem + with CromIamInstrumentation { - private implicit val cs = IO.contextShift(ece) + implicit private val cs = IO.contextShift(ece) override val statusUri = uri"$samBaseUri/status" override val serviceRegistryActor: ActorRef = serviceRegistryActorRef - def isSubmitWhitelisted(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + def isSubmitWhitelisted(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = checkSubmitWhitelist.fold( isSubmitWhitelistedSam(user, cromIamRequest), FailureResponseOrT.pure(true) ) - } def isSubmitWhitelistedSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { val request = HttpRequest( @@ -63,7 +65,7 @@ class SamClient(scheme: String, whitelisted <- response.status match { case StatusCodes.OK => // Does not seem to be already provided? - implicit val entityToBooleanUnmarshaller : Unmarshaller[HttpEntity, Boolean] = + implicit val entityToBooleanUnmarshaller: Unmarshaller[HttpEntity, Boolean] = (Unmarshaller.stringUnmarshaller flatMap Unmarshaller.booleanFromStringUnmarshaller).asScala val unmarshal = IO.fromFuture(IO(Unmarshal(response.entity).to[Boolean])) FailureResponseOrT.right[HttpResponse](unmarshal) @@ -73,8 +75,40 @@ class SamClient(scheme: String, } yield whitelisted } + def isUserEnabledSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + val request = HttpRequest( + method = HttpMethods.GET, + uri = samUserStatusUri, + headers = List[HttpHeader](user.authorization) + ) + + for { + response <- instrumentRequest( + () => Http().singleRequest(request).asFailureResponseOrT, + cromIamRequest, + instrumentationPrefixForSam(getUserEnabledPrefix) + ) + userEnabled <- response.status match { + case StatusCodes.OK => + val unmarshal: IO[UserStatusInfo] = IO.fromFuture(IO(Unmarshal(response.entity).to[UserStatusInfo])) + FailureResponseOrT.right[HttpResponse](unmarshal).map { userInfo => + if (!userInfo.enabled) log.info("Access denied for user {}", user.userId) + userInfo.enabled + } + case _ => + log.error("Could not verify access with Sam for user {}, error was {} {}", + user.userId, + response.status, + response.toString().take(100) + ) + FailureResponseOrT.pure[IO, HttpResponse](false) + } + } yield userEnabled + } + def collectionsForUser(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[List[Collection]] = { - val request = HttpRequest(method = HttpMethods.GET, uri = samBaseCollectionUri, headers = List[HttpHeader](user.authorization)) + val request = + HttpRequest(method = HttpMethods.GET, uri = samBaseCollectionUri, headers = List[HttpHeader](user.authorization)) for { response <- instrumentRequest( @@ -92,24 +126,25 @@ class SamClient(scheme: String, * @return Successful future if the auth is accepted, a Failure otherwise. */ def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Unit] = { val logString = authorizationRequest.action + " access for user " + authorizationRequest.user.userId + - " on a request to " + authorizationRequest.action + " for collection " + authorizationRequest.collection.name + " on a request to " + authorizationRequest.action + " for collection " + authorizationRequest.collection.name - def validateEntityBytes(byteString: ByteString): FailureResponseOrT[Unit] = { + def validateEntityBytes(byteString: ByteString): FailureResponseOrT[Unit] = if (byteString.utf8String == "true") { Monad[FailureResponseOrT].unit } else { log.warning("Sam denied " + logString) FailureResponseOrT[IO, HttpResponse, Unit](IO.raiseError(new SamDenialException)) } - } log.info("Requesting authorization for " + logString) val request = HttpRequest(method = HttpMethods.GET, - uri = samAuthorizeActionUri(authorizationRequest), - headers = List[HttpHeader](authorizationRequest.user.authorization)) + uri = samAuthorizeActionUri(authorizationRequest), + headers = List[HttpHeader](authorizationRequest.user.authorization) + ) for { response <- instrumentRequest( @@ -130,10 +165,7 @@ class SamClient(scheme: String, - If user has the 'add' permission we're ok - else fail the future */ - def requestSubmission(user: User, - collection: Collection, - cromIamRequest: HttpRequest - ): FailureResponseOrT[Unit] = { + def requestSubmission(user: User, collection: Collection, cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { log.info("Verifying user " + user.userId + " can submit a workflow to collection " + collection.name) val createCollection = registerCreation(user, collection, cromIamRequest) @@ -141,15 +173,20 @@ class SamClient(scheme: String, case r if r.status == StatusCodes.NoContent => Monad[FailureResponseOrT].unit case r => FailureResponseOrT[IO, HttpResponse, Unit](IO.raiseError(SamRegisterCollectionException(r.status))) } recoverWith { - case r if r.status == StatusCodes.Conflict => requestAuth(CollectionAuthorizationRequest(user, collection, "add"), cromIamRequest) + case r if r.status == StatusCodes.Conflict => + requestAuth(CollectionAuthorizationRequest(user, collection, "add"), cromIamRequest) case r => FailureResponseOrT[IO, HttpResponse, Unit](IO.raiseError(SamRegisterCollectionException(r.status))) } } protected def registerCreation(user: User, collection: Collection, - cromIamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { - val request = HttpRequest(method = HttpMethods.POST, uri = samRegisterUri(collection), headers = List[HttpHeader](user.authorization)) + cromIamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { + val request = HttpRequest(method = HttpMethods.POST, + uri = samRegisterUri(collection), + headers = List[HttpHeader](user.authorization) + ) instrumentRequest( () => Http().singleRequest(request).asFailureResponseOrT, @@ -158,9 +195,9 @@ class SamClient(scheme: String, ) } - private def samAuthorizeActionUri(authorizationRequest: CollectionAuthorizationRequest) = { - akka.http.scaladsl.model.Uri(s"${samBaseUriForWorkflow(authorizationRequest.collection)}/action/${authorizationRequest.action}") - } + private def samAuthorizeActionUri(authorizationRequest: CollectionAuthorizationRequest) = + akka.http.scaladsl.model + .Uri(s"${samBaseUriForWorkflow(authorizationRequest.collection)}/action/${authorizationRequest.action}") private def samRegisterUri(collection: Collection) = akka.http.scaladsl.model.Uri(samBaseUriForWorkflow(collection)) @@ -170,6 +207,7 @@ class SamClient(scheme: String, private lazy val samBaseResourceUri = s"$samBaseUri/api/resource" private lazy val samBaseCollectionUri = s"$samBaseResourceUri/workflow-collection" private lazy val samSubmitWhitelistUri = s"$samBaseResourceUri/caas/submit/action/get_whitelist" + private lazy val samUserStatusUri = s"$samBaseUri/register/user/v2/self/info" } @@ -178,14 +216,21 @@ object SamClient { class SamDenialException extends Exception("Access Denied") - final case class SamConnectionFailure(phase: String, f: Throwable) extends Exception(s"Unable to connect to Sam during $phase (${f.getMessage})", f) + final case class SamConnectionFailure(phase: String, f: Throwable) + extends Exception(s"Unable to connect to Sam during $phase (${f.getMessage})", f) - final case class SamRegisterCollectionException(errorCode: StatusCode) extends Exception(s"Can't register collection with Sam. Status code: ${errorCode.value}") + final case class SamRegisterCollectionException(errorCode: StatusCode) + extends Exception(s"Can't register collection with Sam. Status code: ${errorCode.value}") final case class CollectionAuthorizationRequest(user: User, collection: Collection, action: String) val SamDenialResponse = HttpResponse(status = StatusCodes.Forbidden, entity = new SamDenialException().getMessage) - def SamRegisterCollectionExceptionResp(statusCode: StatusCode) = HttpResponse(status = statusCode, entity = SamRegisterCollectionException(statusCode).getMessage) + def SamRegisterCollectionExceptionResp(statusCode: StatusCode) = + HttpResponse(status = statusCode, entity = SamRegisterCollectionException(statusCode).getMessage) + + case class UserStatusInfo(adminEnabled: Boolean, enabled: Boolean, userEmail: String, userSubjectId: String) + + implicit val UserStatusInfoFormat: RootJsonFormat[UserStatusInfo] = jsonFormat4(UserStatusInfo) } diff --git a/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala b/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala index 9f5af038b12..b18366490c5 100644 --- a/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala +++ b/CromIAM/src/main/scala/cromiam/server/CromIamServer.scala @@ -15,7 +15,6 @@ import org.broadinstitute.dsde.workbench.util.health.Subsystems.{Cromwell, Sam} import scala.concurrent.{ExecutionContext, ExecutionContextExecutor, Future, Promise} - object CromIamServer extends HttpApp with CromIamApiService with SwaggerService { final val rootConfig: Config = ConfigFactory.load() @@ -35,21 +34,28 @@ object CromIamServer extends HttpApp with CromIamApiService with SwaggerService If there is a reason then leave a comment why there should be two actor systems. https://github.com/broadinstitute/cromwell/issues/3851 */ - CromIamServer.startServer(configuration.cromIamConfig.http.interface, configuration.cromIamConfig.http.port, configuration.cromIamConfig.serverSettings) + CromIamServer.startServer(configuration.cromIamConfig.http.interface, + configuration.cromIamConfig.http.port, + configuration.cromIamConfig.serverSettings + ) } - override implicit val system: ActorSystem = ActorSystem() - override implicit lazy val executor: ExecutionContextExecutor = system.dispatcher - override implicit val materializer: ActorMaterializer = ActorMaterializer() + implicit override val system: ActorSystem = ActorSystem() + implicit override lazy val executor: ExecutionContextExecutor = system.dispatcher + implicit override val materializer: ActorMaterializer = ActorMaterializer() override val log = Logging(system, getClass) override val routes: Route = allRoutes ~ swaggerUiResourceRoute - override val statusService: StatusService = new StatusService(() => Map(Cromwell -> cromwellClient.subsystemStatus(), Sam -> samClient.subsystemStatus())) + override val statusService: StatusService = new StatusService(() => + Map(Cromwell -> cromwellClient.subsystemStatus(), Sam -> samClient.subsystemStatus()) + ) // Override default shutdownsignal which was just "hit return/enter" - override def waitForShutdownSignal(actorSystem: ActorSystem)(implicit executionContext: ExecutionContext): Future[Done] = { + override def waitForShutdownSignal( + actorSystem: ActorSystem + )(implicit executionContext: ExecutionContext): Future[Done] = { val promise = Promise[Done]() sys.addShutdownHook { // we can add anything we want the server to do when someone shutdowns the server (Ctrl-c) diff --git a/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala b/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala index 8d27c7b980d..ce8c5934acd 100644 --- a/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala +++ b/CromIAM/src/main/scala/cromiam/server/config/CromIamServerConfig.scala @@ -15,7 +15,8 @@ import scala.util.{Failure, Success, Try} final case class CromIamServerConfig(cromIamConfig: CromIamConfig, cromwellConfig: ServiceConfig, samConfig: SamClientConfig, - swaggerOauthConfig: SwaggerOauthConfig) + swaggerOauthConfig: SwaggerOauthConfig +) object CromIamServerConfig { def getFromConfig(conf: Config): ErrorOr[CromIamServerConfig] = { @@ -27,22 +28,28 @@ object CromIamServerConfig { (cromIamConfig, cromwellConfig, samConfig, googleConfig) mapN CromIamServerConfig.apply } - private[config] def getValidatedConfigPath[A](conf: Config, path: String, getter: (Config, String) => A, default: Option[A] = None): ErrorOr[A] = { + private[config] def getValidatedConfigPath[A](conf: Config, + path: String, + getter: (Config, String) => A, + default: Option[A] = None + ): ErrorOr[A] = if (conf.hasPath(path)) { Try(getter.apply(conf, path)) match { case Success(s) => s.validNel case Failure(e) => s"Unable to read valid value at '$path': ${e.getMessage}".invalidNel } - } else default match { - case Some(d) => d.validNel - case None => s"Configuration does not have path $path".invalidNel - } - } + } else + default match { + case Some(d) => d.validNel + case None => s"Configuration does not have path $path".invalidNel + } - private[config] implicit final class ValidatingConfig(val conf: Config) extends AnyVal { - def getValidatedString(path: String, default: Option[String] = None): ErrorOr[String] = getValidatedConfigPath(conf, path, (c, p) => c.getString(p), default) + implicit final private[config] class ValidatingConfig(val conf: Config) extends AnyVal { + def getValidatedString(path: String, default: Option[String] = None): ErrorOr[String] = + getValidatedConfigPath(conf, path, (c, p) => c.getString(p), default) def getValidatedInt(path: String): ErrorOr[Int] = getValidatedConfigPath(conf, path, (c, p) => c.getInt(p)) - def getValidatedStringList(path: String): ErrorOr[List[String]] = getValidatedConfigPath[List[String]](conf, path, (c, p) => c.getStringList(p).asScala.toList) + def getValidatedStringList(path: String): ErrorOr[List[String]] = + getValidatedConfigPath[List[String]](conf, path, (c, p) => c.getStringList(p).asScala.toList) } } @@ -50,13 +57,12 @@ final case class CromIamConfig(http: ServiceConfig, serverSettings: ServerSettin object CromIamConfig { - private def getValidatedServerSettings(conf: Config): ErrorOr[ServerSettings] = { + private def getValidatedServerSettings(conf: Config): ErrorOr[ServerSettings] = Try(ServerSettings(conf)) match { case Success(serverSettings) => serverSettings.validNel case Failure(e) => s"Unable to generate server settings from configuration file: ${e.getMessage}".invalidNel } - } private[config] def getFromConfig(conf: Config, basePath: String): ErrorOr[CromIamConfig] = { val serviceConfig = ServiceConfig.getFromConfig(conf, basePath) @@ -94,6 +100,9 @@ object SwaggerOauthConfig { private[config] def getFromConfig(conf: Config, basePath: String): ErrorOr[SwaggerOauthConfig] = { def getValidatedOption(option: String) = conf.getValidatedString(s"$basePath.$option") - (getValidatedOption("client_id"), getValidatedOption("realm"), getValidatedOption("app_name")) mapN SwaggerOauthConfig.apply - } + (getValidatedOption("client_id"), + getValidatedOption("realm"), + getValidatedOption("app_name") + ) mapN SwaggerOauthConfig.apply + } } diff --git a/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala b/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala index 7aa6c0af752..b267b201bb0 100644 --- a/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala +++ b/CromIAM/src/main/scala/cromiam/server/status/StatusCheckedSubsystem.scala @@ -1,7 +1,7 @@ package cromiam.server.status import com.softwaremill.sttp.asynchttpclient.future.AsyncHttpClientFutureBackend -import com.softwaremill.sttp.{Uri, sttp} +import com.softwaremill.sttp.{sttp, Uri} import org.broadinstitute.dsde.workbench.util.health.SubsystemStatus import scala.concurrent.{ExecutionContext, Future} @@ -18,12 +18,11 @@ trait StatusCheckedSubsystem { * Make a call to the status endpoint. If we receive a 200 OK fill in the SubsystemStatus w/ OK = true and no * error messages, otherwise OK = false and include the response body */ - def subsystemStatus()(implicit ec: ExecutionContext): Future[SubsystemStatus] = { + def subsystemStatus()(implicit ec: ExecutionContext): Future[SubsystemStatus] = sttp.get(statusUri).send() map { x => x.body match { case Right(_) => SubsystemStatus(true, None) case Left(errors) => SubsystemStatus(false, Option(List(errors))) } } - } } diff --git a/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala b/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala index 97b16e98a5e..c3f38865224 100644 --- a/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala +++ b/CromIAM/src/main/scala/cromiam/server/status/StatusService.scala @@ -15,13 +15,14 @@ import scala.concurrent.duration._ */ class StatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus]], initialDelay: FiniteDuration = Duration.Zero, - pollInterval: FiniteDuration = 1.minute)(implicit system: ActorSystem, executionContext: ExecutionContext) { + pollInterval: FiniteDuration = 1.minute +)(implicit system: ActorSystem, executionContext: ExecutionContext) { implicit val askTimeout = Timeout(5.seconds) private val healthMonitor = system.actorOf(HealthMonitor.props(Set(Cromwell, Sam))(checkStatus), "HealthMonitorActor") system.scheduler.schedule(initialDelay, pollInterval, healthMonitor, HealthMonitor.CheckAll) - def status(): Future[StatusCheckResponse] = healthMonitor.ask(GetCurrentStatus).asInstanceOf[Future[StatusCheckResponse]] + def status(): Future[StatusCheckResponse] = + healthMonitor.ask(GetCurrentStatus).asInstanceOf[Future[StatusCheckResponse]] } - diff --git a/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala b/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala index 63694599476..d046694b0f9 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/CromIamApiService.scala @@ -16,7 +16,12 @@ import cromiam.auth.{Collection, User} import cromiam.cromwell.CromwellClient import cromiam.instrumentation.CromIamInstrumentation import cromiam.sam.SamClient -import cromiam.sam.SamClient.{CollectionAuthorizationRequest, SamConnectionFailure, SamDenialException, SamDenialResponse} +import cromiam.sam.SamClient.{ + CollectionAuthorizationRequest, + SamConnectionFailure, + SamDenialException, + SamDenialResponse +} import cromiam.server.config.CromIamServerConfig import cromiam.server.status.StatusService import cromiam.webservice.CromIamApiService._ @@ -30,12 +35,13 @@ trait SwaggerService extends SwaggerUiResourceHttpService { } // NB: collection name *must* follow label value rules in cromwell. This needs to be documented somewhere. (although those restrictions are soon to die) -trait CromIamApiService extends RequestSupport - with EngineRouteSupport - with SubmissionSupport - with QuerySupport - with WomtoolRouteSupport - with CromIamInstrumentation { +trait CromIamApiService + extends RequestSupport + with EngineRouteSupport + with SubmissionSupport + with QuerySupport + with WomtoolRouteSupport + with CromIamInstrumentation { implicit val system: ActorSystem implicit def executor: ExecutionContextExecutor @@ -44,23 +50,23 @@ trait CromIamApiService extends RequestSupport protected def rootConfig: Config protected def configuration: CromIamServerConfig - override lazy val serviceRegistryActor: ActorRef = system.actorOf(ServiceRegistryActor.props(rootConfig), "ServiceRegistryActor") + override lazy val serviceRegistryActor: ActorRef = + system.actorOf(ServiceRegistryActor.props(rootConfig), "ServiceRegistryActor") val log: LoggingAdapter - val CromIamExceptionHandler: ExceptionHandler = { - ExceptionHandler { - case e: Exception => - log.error(e, "Request failed {}", e) - complete(HttpResponse(InternalServerError, entity = e.getMessage)) // FIXME: use workbench-model ErrorReport + val CromIamExceptionHandler: ExceptionHandler = + ExceptionHandler { case e: Exception => + log.error(e, "Request failed {}", e) + complete(HttpResponse(InternalServerError, entity = e.getMessage)) // FIXME: use workbench-model ErrorReport } - } lazy val cromwellClient = new CromwellClient(configuration.cromwellConfig.scheme, - configuration.cromwellConfig.interface, - configuration.cromwellConfig.port, - log, - serviceRegistryActor) + configuration.cromwellConfig.interface, + configuration.cromwellConfig.port, + log, + serviceRegistryActor + ) lazy val samClient = new SamClient( configuration.samConfig.http.scheme, @@ -68,7 +74,8 @@ trait CromIamApiService extends RequestSupport configuration.samConfig.http.port, configuration.samConfig.checkSubmitWhitelist, log, - serviceRegistryActor) + serviceRegistryActor + ) val statusService: StatusService @@ -76,12 +83,11 @@ trait CromIamApiService extends RequestSupport workflowLogsRoute ~ abortRoute ~ metadataRoute ~ timingRoute ~ statusRoute ~ backendRoute ~ labelPatchRoute ~ callCacheDiffRoute ~ labelGetRoute ~ releaseHoldRoute - - val allRoutes: Route = handleExceptions(CromIamExceptionHandler) { workflowRoutes ~ engineRoutes ~ womtoolRoutes } + val allRoutes: Route = handleExceptions(CromIamExceptionHandler)(workflowRoutes ~ engineRoutes ~ womtoolRoutes) def abortRoute: Route = path("api" / "workflows" / Segment / Segment / Abort) { (_, workflowId) => post { - extractUserAndRequest { (user, req) => + extractUserAndStrictRequest { (user, req) => logUserWorkflowAction(user, workflowId, Abort) complete { authorizeAbortThenForwardToCromwell(user, workflowId, req).asHttpResponse @@ -90,10 +96,10 @@ trait CromIamApiService extends RequestSupport } } - //noinspection MutatorLikeMethodIsParameterless - def releaseHoldRoute: Route = path("api" / "workflows" / Segment / Segment / ReleaseHold) { (_, workflowId) => + // noinspection MutatorLikeMethodIsParameterless + def releaseHoldRoute: Route = path("api" / "workflows" / Segment / Segment / ReleaseHold) { (_, workflowId) => post { - extractUserAndRequest { (user, req) => + extractUserAndStrictRequest { (user, req) => logUserWorkflowAction(user, workflowId, ReleaseHold) complete { authorizeUpdateThenForwardToCromwell(user, workflowId, req).asHttpResponse @@ -109,35 +115,38 @@ trait CromIamApiService extends RequestSupport def statusRoute: Route = workflowGetRouteWithId("status") def labelGetRoute: Route = workflowGetRouteWithId(Labels) - def labelPatchRoute: Route = { + def labelPatchRoute: Route = path("api" / "workflows" / Segment / Segment / Labels) { (_, workflowId) => patch { - extractUserAndRequest { (user, req) => + extractUserAndStrictRequest { (user, req) => entity(as[String]) { labels => logUserWorkflowAction(user, workflowId, Labels) - validateLabels(Option(labels)) { _ => // Not using the labels, just using this to verify they didn't specify labels we don't want them to - complete { - authorizeUpdateThenForwardToCromwell(user, workflowId, req).asHttpResponse - } + validateLabels(Option(labels)) { + _ => // Not using the labels, just using this to verify they didn't specify labels we don't want them to + complete { + authorizeUpdateThenForwardToCromwell(user, workflowId, req).asHttpResponse + } } } } } } - } def backendRoute: Route = workflowGetRoute("backends") def callCacheDiffRoute: Route = path("api" / "workflows" / Segment / "callcaching" / "diff") { _ => get { - extractUserAndRequest { (user, req) => + extractUserAndStrictRequest { (user, req) => logUserAction(user, "call caching diff") parameterSeq { parameters => val paramMap = parameters.toMap complete { (paramMap.get("workflowA"), paramMap.get("workflowB")) match { case (Some(a), Some(b)) => authorizeReadThenForwardToCromwell(user, List(a, b), req).asHttpResponse - case _ => HttpResponse(status = BadRequest, entity = "Must supply both workflowA and workflowB to the /callcaching/diff endpoint") + case _ => + HttpResponse(status = BadRequest, + entity = "Must supply both workflowA and workflowB to the /callcaching/diff endpoint" + ) } } } @@ -150,11 +159,9 @@ trait CromIamApiService extends RequestSupport */ private def workflowGetRoute(urlSuffix: String): Route = path("api" / "workflows" / Segment / urlSuffix) { _ => get { - extractUserAndRequest { (user, req) => + extractUserAndStrictRequest { (user, req) => logUserAction(user, urlSuffix) - complete { - cromwellClient.forwardToCromwell(req).asHttpResponse - } + forwardIfUserEnabled(user, req, cromwellClient, samClient) } } } @@ -164,31 +171,31 @@ trait CromIamApiService extends RequestSupport */ private def workflowGetRouteWithId(urlSuffix: String): Route = workflowRoute(urlSuffix, get) - private def workflowRoute(urlSuffix: String, method: Directive0): Route = path("api" / "workflows" / Segment / Segment / urlSuffix) { (_, workflowId) => - method { - extractUserAndRequest { (user, req) => - logUserWorkflowAction(user, workflowId, urlSuffix) - complete { - authorizeReadThenForwardToCromwell(user, List(workflowId), req).asHttpResponse + private def workflowRoute(urlSuffix: String, method: Directive0): Route = + path("api" / "workflows" / Segment / Segment / urlSuffix) { (_, workflowId) => + method { + extractUserAndStrictRequest { (user, req) => + logUserWorkflowAction(user, workflowId, urlSuffix) + complete { + authorizeReadThenForwardToCromwell(user, List(workflowId), req).asHttpResponse + } } } } - } private def authorizeThenForwardToCromwell(user: User, workflowIds: List[String], action: String, request: HttpRequest, - cromwellClient: CromwellClient): - FailureResponseOrT[HttpResponse] = { - def authForCollection(collection: Collection): FailureResponseOrT[Unit] = { + cromwellClient: CromwellClient + ): FailureResponseOrT[HttpResponse] = { + def authForCollection(collection: Collection): FailureResponseOrT[Unit] = samClient.requestAuth(CollectionAuthorizationRequest(user, collection, action), request) mapErrorWith { case e: SamDenialException => IO.raiseError(e) case e => log.error(e, "Unable to connect to Sam {}", e) IO.raiseError(SamConnectionFailure("authorization", e)) } - } val cromwellResponseT = for { rootWorkflowIds <- workflowIds.traverse(cromwellClient.getRootWorkflow(_, user, request)) @@ -211,31 +218,29 @@ trait CromIamApiService extends RequestSupport private def authorizeReadThenForwardToCromwell(user: User, workflowIds: List[String], request: HttpRequest - ): FailureResponseOrT[HttpResponse] = { - authorizeThenForwardToCromwell( - user = user, - workflowIds = workflowIds, - action = "view", - request = request, - cromwellClient = cromwellClient) - } + ): FailureResponseOrT[HttpResponse] = + authorizeThenForwardToCromwell(user = user, + workflowIds = workflowIds, + action = "view", + request = request, + cromwellClient = cromwellClient + ) private def authorizeUpdateThenForwardToCromwell(user: User, workflowId: String, request: HttpRequest - ): FailureResponseOrT[HttpResponse] = { - authorizeThenForwardToCromwell( - user = user, - workflowIds = List(workflowId), - action = "update", - request = request, - cromwellClient = cromwellClient) - } + ): FailureResponseOrT[HttpResponse] = + authorizeThenForwardToCromwell(user = user, + workflowIds = List(workflowId), + action = "update", + request = request, + cromwellClient = cromwellClient + ) private def authorizeAbortThenForwardToCromwell(user: User, workflowId: String, request: HttpRequest - ): FailureResponseOrT[HttpResponse] = { + ): FailureResponseOrT[HttpResponse] = // Do all the authing for the abort with "this" cromwell instance (cromwellClient), but the actual abort command // must go to the dedicated abort server (cromwellAbortClient). authorizeThenForwardToCromwell( @@ -245,13 +250,11 @@ trait CromIamApiService extends RequestSupport request = request, cromwellClient = cromwellClient ) - } private def logUserAction(user: User, action: String) = log.info("User " + user.userId + " requesting " + action) - private def logUserWorkflowAction(user: User, wfId: String, action: String) = { + private def logUserWorkflowAction(user: User, wfId: String, action: String) = log.info("User " + user.userId + " requesting " + action + " with " + wfId) - } } object CromIamApiService { diff --git a/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala b/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala index 5719bda3479..42f69ee6449 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/EngineRouteSupport.scala @@ -13,7 +13,6 @@ import org.broadinstitute.dsde.workbench.util.health.StatusJsonSupport._ import scala.concurrent.ExecutionContextExecutor - trait EngineRouteSupport extends RequestSupport with SprayJsonSupport { val statusService: StatusService val cromwellClient: CromwellClient @@ -25,7 +24,7 @@ trait EngineRouteSupport extends RequestSupport with SprayJsonSupport { def versionRoute: Route = path("engine" / Segment / "version") { _ => get { extractStrictRequest { req => - complete { cromwellClient.forwardToCromwell(req).asHttpResponse } + complete(cromwellClient.forwardToCromwell(req).asHttpResponse) } } } @@ -39,10 +38,11 @@ trait EngineRouteSupport extends RequestSupport with SprayJsonSupport { } } - def statsRoute: Route = path("engine" / Segment / "stats") { _ => complete(CromIamStatsForbidden) } + def statsRoute: Route = path("engine" / Segment / "stats")(_ => complete(CromIamStatsForbidden)) } object EngineRouteSupport { - private[webservice] val CromIamStatsForbidden = HttpResponse(status = Forbidden, entity = "CromIAM does not allow access to the /stats endpoint") + private[webservice] val CromIamStatsForbidden = + HttpResponse(status = Forbidden, entity = "CromIAM does not allow access to the /stats endpoint") } diff --git a/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala b/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala index cddabe74a57..68ad1f21230 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/QuerySupport.scala @@ -43,7 +43,7 @@ trait QuerySupport extends RequestSupport { post { preprocessQuery { (user, collections, request) => processLabelsForPostQuery(user, collections) { entity => - complete { cromwellClient.forwardToCromwell(request.withEntity(entity)).asHttpResponse } + complete(cromwellClient.forwardToCromwell(request.withEntity(entity)).asHttpResponse) } } } @@ -54,8 +54,8 @@ trait QuerySupport extends RequestSupport { * retrieves the collections for the user, grabs the underlying HttpRequest and forwards it on to the specific * directive */ - private def preprocessQuery: Directive[(User, List[Collection], HttpRequest)] = { - extractUserAndRequest tflatMap { case (user, cromIamRequest) => + private def preprocessQuery: Directive[(User, List[Collection], HttpRequest)] = + extractUserAndStrictRequest tflatMap { case (user, cromIamRequest) => log.info("Received query " + cromIamRequest.method.value + " request for user " + user.userId) onComplete(samClient.collectionsForUser(user, cromIamRequest).value.unsafeToFuture()) flatMap { @@ -71,13 +71,12 @@ trait QuerySupport extends RequestSupport { throw new RuntimeException(s"Unable to look up collections for user ${user.userId}: ${e.getMessage}", e) } } - } /** * Will verify that none of the GET query parameters are specifying the collection label, and then tack * on query parameters for the user's collections on to the query URI */ - private def processLabelsForGetQuery(user: User, collections: List[Collection]): Directive1[Uri] = { + private def processLabelsForGetQuery(user: User, collections: List[Collection]): Directive1[Uri] = extractUri flatMap { uri => val query = uri.query() @@ -95,7 +94,6 @@ trait QuerySupport extends RequestSupport { provide(uri.withQuery(newQueryBuilder.result())) } - } /** * Will verify that none of the POSTed query parameters are specifying the collection label, and then tack @@ -115,7 +113,7 @@ trait QuerySupport extends RequestSupport { case jsObject if jsObject.fields.keySet.exists(key => key.equalsIgnoreCase(LabelOrKey)) => jsObject.fields.values.map(_.convertTo[String]) } - ).flatten + ).flatten // DO NOT REMOVE THE NEXT LINE WITHOUT READING THE SCALADOC ON ensureNoLabelOrs ensureNoLabelOrs(user, labelOrs) @@ -152,12 +150,11 @@ trait QuerySupport extends RequestSupport { * - https://github.com/persvr/rql#rql-rules * - https://github.com/jirutka/rsql-parser#grammar-and-semantic */ - protected[this] def ensureNoLabelOrs(user: User, labelOrs: Iterable[String]): Unit = { + protected[this] def ensureNoLabelOrs(user: User, labelOrs: Iterable[String]): Unit = labelOrs.toList match { case Nil => () case head :: tail => throw new LabelContainsOrException(user, NonEmptyList(head, tail)) } - } /** * Returns the user's collections as a set of labels @@ -169,12 +166,14 @@ trait QuerySupport extends RequestSupport { } object QuerySupport { - final case class InvalidQueryException(e: Throwable) extends - Exception(s"Invalid JSON in query POST body: ${e.getMessage}", e) - - final class LabelContainsOrException(val user: User, val labelOrs: NonEmptyList[String]) extends - Exception(s"User ${user.userId} submitted a labels query containing an OR which CromIAM is blocking: " + - labelOrs.toList.mkString("LABELS CONTAIN '", "' OR LABELS CONTAIN '", "'")) + final case class InvalidQueryException(e: Throwable) + extends Exception(s"Invalid JSON in query POST body: ${e.getMessage}", e) + + final class LabelContainsOrException(val user: User, val labelOrs: NonEmptyList[String]) + extends Exception( + s"User ${user.userId} submitted a labels query containing an OR which CromIAM is blocking: " + + labelOrs.toList.mkString("LABELS CONTAIN '", "' OR LABELS CONTAIN '", "'") + ) val LabelAndKey = "label" val LabelOrKey = "labelor" diff --git a/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala b/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala index 12b231485f7..9ccb08a7b28 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/RequestSupport.scala @@ -6,30 +6,58 @@ import akka.http.scaladsl.server.Directives._ import akka.http.scaladsl.server._ import cromiam.auth.User import org.broadinstitute.dsde.workbench.model.WorkbenchUserId +import akka.http.scaladsl.model.HttpResponse +import akka.http.scaladsl.server.Directives.{authorize, complete, onComplete} +import akka.http.scaladsl.server.Route +import cromiam.cromwell.CromwellClient +import cromiam.sam.SamClient + +import scala.util.{Failure, Success} trait RequestSupport { - def extractStrictRequest: Directive1[HttpRequest] = { + def extractStrictRequest: Directive1[HttpRequest] = toStrictEntity(Timeout) tflatMap { _ => extractRequest flatMap { request => provide(request) } } - } /** * Obtain both the user id header from the proxy as well as the bearer token and pass that back * into the route logic as a User object */ - def extractUser: Directive1[User] = { - (headerValueByName("OIDC_CLAIM_user_id") & headerValuePF { case a: Authorization => a }) tmap { case (userId, auth) => - User(WorkbenchUserId(userId), auth) + def extractUser: Directive1[User] = + (headerValueByName("OIDC_CLAIM_user_id") & headerValuePF { case a: Authorization => a }) tmap { + case (userId, auth) => + User(WorkbenchUserId(userId), auth) } - } - def extractUserAndRequest: Directive[(User, HttpRequest)] = { + def extractUserAndStrictRequest: Directive[(User, HttpRequest)] = for { user <- extractUser request <- extractStrictRequest } yield (user, request) + + def forwardIfUserEnabled(user: User, + req: HttpRequest, + cromwellClient: CromwellClient, + samClient: SamClient + ): Route = { + import cromwell.api.model.EnhancedFailureResponseOrHttpResponseT + + onComplete(samClient.isUserEnabledSam(user, req).value.unsafeToFuture()) { + case Success(Left(httpResponse: HttpResponse)) => complete(httpResponse) + case Success(Right(isEnabled: Boolean)) => + authorize(isEnabled) { + complete { + cromwellClient.forwardToCromwell(req).asHttpResponse + } + } + case Failure(e) => + val message = + s"Unable to look up enablement status for user ${user.userId}: ${e.getMessage}. Please try again later." + throw new RuntimeException(message, e) + } } + } diff --git a/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala b/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala index c1f5477475b..79c66a77313 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/SubmissionSupport.scala @@ -7,7 +7,7 @@ import akka.http.scaladsl.server._ import akka.stream.ActorMaterializer import akka.util.ByteString import cats.effect.IO -import cromiam.auth.Collection.{CollectionLabelName, LabelsKey, validateLabels} +import cromiam.auth.Collection.{validateLabels, CollectionLabelName, LabelsKey} import cromiam.auth.{Collection, User} import cromiam.cromwell.CromwellClient import cromiam.sam.SamClient @@ -31,7 +31,7 @@ trait SubmissionSupport extends RequestSupport { // FIXME - getting pathPrefix to shrink this keeps hosing up, there's gotta be some way to do this def submitRoute: Route = (path("api" / "workflows" / Segment) | path("api" / "workflows" / Segment / "batch")) { _ => post { - extractUserAndRequest { (user, request) => + extractUserAndStrictRequest { (user, request) => log.info("Received submission request from user " + user.userId) onComplete(samClient.isSubmitWhitelisted(user, request).value.unsafeToFuture()) { case Success(Left(httpResponse)) => complete(httpResponse) @@ -57,16 +57,18 @@ trait SubmissionSupport extends RequestSupport { private def forwardSubmissionToCromwell(user: User, collection: Collection, - submissionRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { - log.info("Forwarding submission request for " + user.userId + " with collection " + collection.name + " to Cromwell") + submissionRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { + log.info( + "Forwarding submission request for " + user.userId + " with collection " + collection.name + " to Cromwell" + ) - def registerWithSam(collection: Collection, httpRequest: HttpRequest): FailureResponseOrT[Unit] = { + def registerWithSam(collection: Collection, httpRequest: HttpRequest): FailureResponseOrT[Unit] = samClient.requestSubmission(user, collection, httpRequest) mapErrorWith { case e: SamDenialException => IO.raiseError(e) case SamRegisterCollectionException(statusCode) => IO.raiseError(SamRegisterCollectionException(statusCode)) case e => IO.raiseError(SamConnectionFailure("new workflow registration", e)) } - } FailureResponseOrT( (for { @@ -81,36 +83,34 @@ trait SubmissionSupport extends RequestSupport { } object SubmissionSupport { - def extractCollection(user: User): Directive1[Collection] = { + def extractCollection(user: User): Directive1[Collection] = formField(CollectionNameKey.?) map { maybeCollectionName => maybeCollectionName.map(Collection(_)).getOrElse(Collection.forUser(user)) } - } - def extractSubmission(user: User): Directive1[WorkflowSubmission] = { + def extractSubmission(user: User): Directive1[WorkflowSubmission] = ( extractCollection(user) & - formFields(( - WorkflowSourceKey.?, - WorkflowUrlKey.?, - WorkflowTypeKey.?, - WorkflowTypeVersionKey.?, - WorkflowInputsKey.?, - WorkflowOptionsKey.?, - WorkflowOnHoldKey.as[Boolean].?, - WorkflowDependenciesKey.as[ByteString].?)) & - extractLabels & - extractInputAux + formFields( + (WorkflowSourceKey.?, + WorkflowUrlKey.?, + WorkflowTypeKey.?, + WorkflowTypeVersionKey.?, + WorkflowInputsKey.?, + WorkflowOptionsKey.?, + WorkflowOnHoldKey.as[Boolean].?, + WorkflowDependenciesKey.as[ByteString].? + ) + ) & + extractLabels & + extractInputAux ).as(WorkflowSubmission) - } - def extractLabels: Directive1[Option[Map[String, JsValue]]] = { + def extractLabels: Directive1[Option[Map[String, JsValue]]] = formField(LabelsKey.?) flatMap validateLabels - } - def extractInputAux: Directive1[Map[String, String]] = { + def extractInputAux: Directive1[Map[String, String]] = formFieldMap.map(_.view.filterKeys(_.startsWith(WorkflowInputsAuxPrefix)).toMap) - } // FIXME: Much like CromwellClient see if there are ways of unifying this a bit w/ the mothership final case class WorkflowSubmission(collection: Collection, @@ -123,32 +123,61 @@ object SubmissionSupport { workflowOnHold: Option[Boolean], workflowDependencies: Option[ByteString], origLabels: Option[Map[String, JsValue]], - workflowInputsAux: Map[String, String]) { + workflowInputsAux: Map[String, String] + ) { // For auto-validation, if origLabels defined, can't have CaaS collection label set. Was checked previously, but ... require(origLabels.forall(!_.keySet.contains(CollectionLabelName))) // Inject the collection name into the labels and convert to a String private val collectionLabels = Map(CollectionLabelName -> JsString(collection.name)) - private val labels: String = JsObject(origLabels.map(o => o ++ collectionLabels).getOrElse(collectionLabels)).toString + private val labels: String = JsObject( + origLabels.map(o => o ++ collectionLabels).getOrElse(collectionLabels) + ).toString val entity: MessageEntity = { - val sourcePart = workflowSource map { s => Multipart.FormData.BodyPart(WorkflowSourceKey, HttpEntity(MediaTypes.`application/json`, s)) } - val urlPart = workflowUrl map { u => Multipart.FormData.BodyPart(WorkflowUrlKey, HttpEntity(MediaTypes.`application/json`, u))} - val typePart = workflowType map { t => Multipart.FormData.BodyPart(WorkflowTypeKey, HttpEntity(MediaTypes.`application/json`, t)) } - val typeVersionPart = workflowTypeVersion map { v => Multipart.FormData.BodyPart(WorkflowTypeVersionKey, HttpEntity(MediaTypes.`application/json`, v)) } - val inputsPart = workflowInputs map { i => Multipart.FormData.BodyPart(WorkflowInputsKey, HttpEntity(MediaTypes.`application/json`, i)) } - val optionsPart = workflowOptions map { o => Multipart.FormData.BodyPart(WorkflowOptionsKey, HttpEntity(MediaTypes.`application/json`, o)) } - val importsPart = workflowDependencies map { d => Multipart.FormData.BodyPart(WorkflowDependenciesKey, HttpEntity(MediaTypes.`application/octet-stream`, d)) } - val onHoldPart = workflowOnHold map { h => Multipart.FormData.BodyPart(WorkflowOnHoldKey, HttpEntity(h.toString)) } + val sourcePart = workflowSource map { s => + Multipart.FormData.BodyPart(WorkflowSourceKey, HttpEntity(MediaTypes.`application/json`, s)) + } + val urlPart = workflowUrl map { u => + Multipart.FormData.BodyPart(WorkflowUrlKey, HttpEntity(MediaTypes.`application/json`, u)) + } + val typePart = workflowType map { t => + Multipart.FormData.BodyPart(WorkflowTypeKey, HttpEntity(MediaTypes.`application/json`, t)) + } + val typeVersionPart = workflowTypeVersion map { v => + Multipart.FormData.BodyPart(WorkflowTypeVersionKey, HttpEntity(MediaTypes.`application/json`, v)) + } + val inputsPart = workflowInputs map { i => + Multipart.FormData.BodyPart(WorkflowInputsKey, HttpEntity(MediaTypes.`application/json`, i)) + } + val optionsPart = workflowOptions map { o => + Multipart.FormData.BodyPart(WorkflowOptionsKey, HttpEntity(MediaTypes.`application/json`, o)) + } + val importsPart = workflowDependencies map { d => + Multipart.FormData.BodyPart(WorkflowDependenciesKey, HttpEntity(MediaTypes.`application/octet-stream`, d)) + } + val onHoldPart = workflowOnHold map { h => + Multipart.FormData.BodyPart(WorkflowOnHoldKey, HttpEntity(h.toString)) + } val labelsPart = Multipart.FormData.BodyPart(LabelsKey, HttpEntity(MediaTypes.`application/json`, labels)) - val parts = List(sourcePart, urlPart, typePart, typeVersionPart, inputsPart, optionsPart, importsPart, onHoldPart, Option(labelsPart)).flatten ++ auxParts + val parts = List(sourcePart, + urlPart, + typePart, + typeVersionPart, + inputsPart, + optionsPart, + importsPart, + onHoldPart, + Option(labelsPart) + ).flatten ++ auxParts Multipart.FormData(parts: _*).toEntity() } - private def auxParts = { - workflowInputsAux map { case (k, v) => Multipart.FormData.BodyPart(k, HttpEntity(MediaTypes.`application/json`, v)) } - } + private def auxParts = + workflowInputsAux map { case (k, v) => + Multipart.FormData.BodyPart(k, HttpEntity(MediaTypes.`application/json`, v)) + } } // FIXME: Unify these w/ Cromwell.PartialWorkflowSources (via common?) diff --git a/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala b/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala index 9fed12ca163..874949a5244 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/SwaggerUiHttpService.scala @@ -27,7 +27,7 @@ trait SwaggerUiHttpService extends Directives { s"META-INF/resources/webjars/swagger-ui/$swaggerUiVersion" } - private val serveIndex: server.Route = { + private val serveIndex: server.Route = mapResponseEntity { entityFromJar => entityFromJar.transformDataBytes(Flow.fromFunction[ByteString, ByteString] { original: ByteString => ByteString(rewriteSwaggerIndex(original.utf8String)) @@ -35,14 +35,13 @@ trait SwaggerUiHttpService extends Directives { } { getFromResource(s"$resourceDirectory/index.html") } - } /** * Serves up the swagger UI only. Redirects requests to the root of the UI path to the index.html. * * @return Route serving the swagger UI. */ - final def swaggerUiRoute: Route = { + final def swaggerUiRoute: Route = pathEndOrSingleSlash { get { serveIndex @@ -68,16 +67,15 @@ trait SwaggerUiHttpService extends Directives { } } - } /** Rewrite the swagger index.html. Default passes through the origin data. */ protected def rewriteSwaggerIndex(original: String): String = { val swaggerOptions = s""" - | validatorUrl: null, - | apisSorter: "alpha", - | oauth2RedirectUrl: window.location.origin + "/swagger/oauth2-redirect.html", - | operationsSorter: "alpha" + | validatorUrl: null, + | apisSorter: "alpha", + | oauth2RedirectUrl: window.location.origin + "/swagger/oauth2-redirect.html", + | operationsSorter: "alpha" """.stripMargin val initOAuthOriginal = "window.ui = ui" @@ -94,7 +92,6 @@ trait SwaggerUiHttpService extends Directives { |$initOAuthOriginal |""".stripMargin - original .replace(initOAuthOriginal, initOAuthReplacement) .replace("""url: "https://petstore.swagger.io/v2/swagger.json"""", "url: 'cromiam.yaml'") @@ -109,6 +106,7 @@ trait SwaggerUiHttpService extends Directives { * swagger UI, but defaults to "yaml". This is an alternative to spray-swagger's SwaggerHttpService. */ trait SwaggerResourceHttpService { + /** * @return The directory for the resource under the classpath, and in the url */ @@ -134,7 +132,8 @@ trait SwaggerResourceHttpService { */ final def swaggerResourceRoute: Route = { // Serve CromIAM API docs from either `/swagger/cromiam.yaml` or just `cromiam.yaml`. - val swaggerDocsDirective = path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType") + val swaggerDocsDirective = + path(separateOnSlashes(swaggerDocsPath)) | path(s"$swaggerServiceName.$swaggerResourceType") val route = get { swaggerDocsDirective { // Return /uiPath/serviceName.resourceType from the classpath resources. diff --git a/CromIAM/src/main/scala/cromiam/webservice/WomtoolRouteSupport.scala b/CromIAM/src/main/scala/cromiam/webservice/WomtoolRouteSupport.scala index 671d4a76543..a6098b1fae0 100644 --- a/CromIAM/src/main/scala/cromiam/webservice/WomtoolRouteSupport.scala +++ b/CromIAM/src/main/scala/cromiam/webservice/WomtoolRouteSupport.scala @@ -2,20 +2,18 @@ package cromiam.webservice import akka.http.scaladsl.server.Directives._ import cromiam.cromwell.CromwellClient -import cromwell.api.model._ +import cromiam.sam.SamClient trait WomtoolRouteSupport extends RequestSupport { // When this trait is mixed into `CromIamApiService` the value of `cromwellClient` is the reader (non-abort) address val cromwellClient: CromwellClient + val samClient: SamClient val womtoolRoutes = path("api" / "womtool" / Segment / "describe") { _ => post { - extractStrictRequest { req => - complete { - // This endpoint requires authn which it gets for free from the proxy, does not care about authz - cromwellClient.forwardToCromwell(req).asHttpResponse - } + extractUserAndStrictRequest { (user, req) => + forwardIfUserEnabled(user, req, cromwellClient, samClient) } } } diff --git a/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala b/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala index 5f3f4fd3791..0ee89b9bbb6 100644 --- a/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala +++ b/CromIAM/src/test/scala/cromiam/auth/CollectionSpec.scala @@ -42,7 +42,6 @@ class CollectionSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers } } - behavior of "validateLabels" it should "not throw exception when labels are valid" in { val labels = """{"key-1":"foo","key-2":"bar"}""" diff --git a/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala b/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala index ce6f16935d6..303aff01dc4 100644 --- a/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala +++ b/CromIAM/src/test/scala/cromiam/cromwell/CromwellClientSpec.scala @@ -39,25 +39,35 @@ class CromwellClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfter } "CromwellClient" should "eventually return a subworkflow's root workflow id" in { - cromwellClient.getRootWorkflow(SubworkflowId.id.toString, FictitiousUser, fakeHttpRequest).map(w => assert(w == RootWorkflowId.id.toString)) - .asIo.unsafeToFuture() + cromwellClient + .getRootWorkflow(SubworkflowId.id.toString, FictitiousUser, fakeHttpRequest) + .map(w => assert(w == RootWorkflowId.id.toString)) + .asIo + .unsafeToFuture() } it should "eventually return a top level workflow's ID when requesting root workflow id" in { - cromwellClient.getRootWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest).map(w => assert(w == RootWorkflowId.id.toString)) - .asIo.unsafeToFuture() + cromwellClient + .getRootWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest) + .map(w => assert(w == RootWorkflowId.id.toString)) + .asIo + .unsafeToFuture() } it should "properly fetch the collection for a workflow with a collection name" in { - cromwellClient.collectionForWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest).map(c => - assert(c.name == CollectionName) - ).asIo.unsafeToFuture() + cromwellClient + .collectionForWorkflow(RootWorkflowId.id.toString, FictitiousUser, fakeHttpRequest) + .map(c => assert(c.name == CollectionName)) + .asIo + .unsafeToFuture() } it should "throw an exception if the workflow doesn't have a collection" in { recoverToExceptionIf[IllegalArgumentException] { - cromwellClient.collectionForWorkflow(WorkflowIdWithoutCollection.id.toString, FictitiousUser, fakeHttpRequest) - .asIo.unsafeToFuture() + cromwellClient + .collectionForWorkflow(WorkflowIdWithoutCollection.id.toString, FictitiousUser, fakeHttpRequest) + .asIo + .unsafeToFuture() } map { exception => assert(exception.getMessage == s"Workflow $WorkflowIdWithoutCollection has no associated collection") } @@ -65,24 +75,25 @@ class CromwellClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfter } object CromwellClientSpec { - final class MockCromwellClient()(implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender) { + final class MockCromwellClient()(implicit + system: ActorSystem, + ece: ExecutionContextExecutor, + materializer: ActorMaterializer + ) extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender) { override val cromwellApiClient: CromwellApiClient = new MockCromwellApiClient() override def sendTimingApi(statsDPath: InstrumentationPath, timing: FiniteDuration, prefixToStatsd: NonEmptyList[String] - ): Unit = () + ): Unit = () } final class MockCromwellApiClient()(implicit actorSystem: ActorSystem, materializer: ActorMaterializer) - extends CromwellApiClient(new URL("http://foo.com"), "bar") { + extends CromwellApiClient(new URL("http://foo.com"), "bar") { - - override def labels(workflowId: WorkflowId, headers: List[HttpHeader] = defaultHeaders) - (implicit ec: ExecutionContext): FailureResponseOrT[WorkflowLabels] = { + override def labels(workflowId: WorkflowId, headers: List[HttpHeader] = defaultHeaders)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowLabels] = if (workflowId == RootWorkflowId) { FailureResponseOrT.pure(FictitiousWorkflowLabelsWithCollection) } else if (workflowId == WorkflowIdWithoutCollection) { @@ -92,18 +103,17 @@ object CromwellClientSpec { IO.raiseError(new RuntimeException("Unexpected workflow ID sent to MockCromwellApiClient")) } } - } override def metadata(workflowId: WorkflowId, - args: Option[Map[String, List[String]]] = None, - headers: List[HttpHeader] = defaultHeaders - )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = { + args: Option[Map[String, List[String]]] = None, + headers: List[HttpHeader] = defaultHeaders + )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = if (workflowId == RootWorkflowId) FailureResponseOrT.pure(RootWorkflowMetadata) else if (workflowId == SubworkflowId) FailureResponseOrT.pure(SubWorkflowMetadata) - else FailureResponseOrT[IO, HttpResponse, WorkflowMetadata] { - IO.raiseError(new RuntimeException("Unexpected workflow ID sent to MockCromwellApiClient")) - } - } + else + FailureResponseOrT[IO, HttpResponse, WorkflowMetadata] { + IO.raiseError(new RuntimeException("Unexpected workflow ID sent to MockCromwellApiClient")) + } } private val SubworkflowId = WorkflowId.fromString("58114f5c-f439-4488-8d73-092273cf92d9") @@ -126,7 +136,8 @@ object CromwellClientSpec { }""") val CollectionName = "foo" - val FictitiousWorkflowLabelsWithCollection = WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("caas-collection-name" -> JsString(CollectionName)))) - val FictitiousWorkflowLabelsWithoutCollection = WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("something" -> JsString("foo")))) + val FictitiousWorkflowLabelsWithCollection = + WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("caas-collection-name" -> JsString(CollectionName)))) + val FictitiousWorkflowLabelsWithoutCollection = + WorkflowLabels(RootWorkflowId.id.toString, JsObject(Map("something" -> JsString("foo")))) } - diff --git a/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala b/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala index 40f0cda2e86..95869718fb5 100644 --- a/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala +++ b/CromIAM/src/test/scala/cromiam/sam/SamClientSpec.scala @@ -28,7 +28,8 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { HttpResponse(StatusCodes.InternalServerError, entity = HttpEntity("expected error")) private val authorization = Authorization(OAuth2BearerToken("my-token")) - private val authorizedUserWithCollection = User(WorkbenchUserId(MockSamClient.AuthorizedUserCollectionStr), authorization) + private val authorizedUserWithCollection = + User(WorkbenchUserId(MockSamClient.AuthorizedUserCollectionStr), authorization) private val unauthorizedUserWithNoCollection = User(WorkbenchUserId(MockSamClient.UnauthorizedUserCollectionStr), authorization) private val notWhitelistedUser = User(WorkbenchUserId(MockSamClient.NotWhitelistedUser), authorization) @@ -47,25 +48,25 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { super.afterAll() } - behavior of "SamClient" it should "return true if user is whitelisted" in { val samClient = new MockSamClient() - samClient.isSubmitWhitelisted(authorizedUserWithCollection, emptyHttpRequest).map(v => assert(v)) - .asIo.unsafeToFuture() + samClient + .isSubmitWhitelisted(authorizedUserWithCollection, emptyHttpRequest) + .map(v => assert(v)) + .asIo + .unsafeToFuture() } it should "return false if user is not whitelisted" in { val samClient = new MockSamClient() - samClient.isSubmitWhitelisted(notWhitelistedUser, emptyHttpRequest).map(v => assert(!v)) - .asIo.unsafeToFuture() + samClient.isSubmitWhitelisted(notWhitelistedUser, emptyHttpRequest).map(v => assert(!v)).asIo.unsafeToFuture() } it should "return sam errors while checking is whitelisted" in { val samClient = new MockSamClient() { - override def isSubmitWhitelistedSam(user: User, cromiamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + override def isSubmitWhitelistedSam(user: User, cromiamRequest: HttpRequest): FailureResponseOrT[Boolean] = MockSamClient.returnResponse(expectedErrorResponse) - } } samClient.isSubmitWhitelisted(notWhitelistedUser, emptyHttpRequest).value.unsafeToFuture() map { _ should be(Left(expectedErrorResponse)) @@ -74,32 +75,33 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { it should "eventually return the collection(s) of user" in { val samClient = new MockSamClient() - samClient.collectionsForUser(authorizedUserWithCollection, emptyHttpRequest).map(collectionList => - assert(collectionList == MockSamClient.UserCollectionList) - ).asIo.unsafeToFuture() + samClient + .collectionsForUser(authorizedUserWithCollection, emptyHttpRequest) + .map(collectionList => assert(collectionList == MockSamClient.UserCollectionList)) + .asIo + .unsafeToFuture() } it should "fail if user doesn't have any collections" in { val samClient = new MockSamClient() recoverToExceptionIf[Exception] { - samClient.collectionsForUser(unauthorizedUserWithNoCollection, emptyHttpRequest) - .asIo.unsafeToFuture() - } map(exception => - assert(exception.getMessage == s"Unable to look up collections for user ${unauthorizedUserWithNoCollection.userId.value}!") + samClient.collectionsForUser(unauthorizedUserWithNoCollection, emptyHttpRequest).asIo.unsafeToFuture() + } map (exception => + assert( + exception.getMessage == s"Unable to look up collections for user ${unauthorizedUserWithNoCollection.userId.value}!" + ) ) } it should "return true if user is authorized to perform action on collection" in { val samClient = new MockSamClient() - samClient.requestAuth(authorizedCollectionRequest, emptyHttpRequest).map(_ => succeed) - .asIo.unsafeToFuture() + samClient.requestAuth(authorizedCollectionRequest, emptyHttpRequest).map(_ => succeed).asIo.unsafeToFuture() } it should "throw SamDenialException if user is not authorized to perform action on collection" in { val samClient = new MockSamClient() recoverToExceptionIf[SamDenialException] { - samClient.requestAuth(unauthorizedCollectionRequest, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient.requestAuth(unauthorizedCollectionRequest, emptyHttpRequest).asIo.unsafeToFuture() } map { exception => assert(exception.getMessage == "Access Denied") } @@ -107,15 +109,21 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { it should "register collection to Sam if user has authorization to create/add to collection" in { val samClient = new MockSamClient() - samClient.requestSubmission(authorizedUserWithCollection, authorizedCollection, emptyHttpRequest).map(_ => succeed) - .asIo.unsafeToFuture() + samClient + .requestSubmission(authorizedUserWithCollection, authorizedCollection, emptyHttpRequest) + .map(_ => succeed) + .asIo + .unsafeToFuture() } it should "throw SamRegisterCollectionException if user doesn't have authorization to create/add to collection" in { val samClient = new MockSamClient() recoverToExceptionIf[SamRegisterCollectionException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest).map(_ => succeed) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .map(_ => succeed) + .asIo + .unsafeToFuture() } map { exception => assert(exception.getMessage == "Can't register collection with Sam. Status code: 400 Bad Request") } @@ -125,15 +133,16 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val conflictResponse = HttpResponse(StatusCodes.Conflict, entity = HttpEntity("expected conflict")) returnResponse(conflictResponse) } override def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromiamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[Unit] = Monad[FailureResponseOrT].unit - } } samClient .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) @@ -146,19 +155,22 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val conflictResponse = HttpResponse(StatusCodes.Conflict, entity = HttpEntity("expected conflict")) returnResponse(conflictResponse) } override def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromiamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[Unit] = returnResponse(expectedErrorResponse) - } } recoverToExceptionIf[UnsuccessfulRequestException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .asIo + .unsafeToFuture() } map { exception => assert(exception.getMessage == "expected error") } @@ -168,14 +180,17 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val unexpectedOkResponse = HttpResponse(StatusCodes.OK, entity = HttpEntity("elided ok message")) returnResponse(unexpectedOkResponse) } } recoverToExceptionIf[SamRegisterCollectionException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .asIo + .unsafeToFuture() } map { exception => exception.getMessage should be("Can't register collection with Sam. Status code: 200 OK") } @@ -185,14 +200,17 @@ class SamClientSpec extends AsyncFlatSpec with Matchers with BeforeAndAfterAll { val samClient = new BaseMockSamClient() { override protected def registerCreation(user: User, collection: Collection, - cromiamRequest: HttpRequest): FailureResponseOrT[HttpResponse] = { + cromiamRequest: HttpRequest + ): FailureResponseOrT[HttpResponse] = { val unexpectedFailureResponse = HttpResponse(StatusCodes.ImATeapot, entity = HttpEntity("elided error message")) returnResponse(unexpectedFailureResponse) } } recoverToExceptionIf[SamRegisterCollectionException] { - samClient.requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) - .asIo.unsafeToFuture() + samClient + .requestSubmission(unauthorizedUserWithNoCollection, unauthorizedCollection, emptyHttpRequest) + .asIo + .unsafeToFuture() } map { exception => exception.getMessage should be("Can't register collection with Sam. Status code: 418 I'm a teapot") } diff --git a/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala b/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala index 9010da8bda0..c7866c68089 100644 --- a/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala +++ b/CromIAM/src/test/scala/cromiam/server/status/MockStatusService.scala @@ -6,8 +6,10 @@ import org.broadinstitute.dsde.workbench.util.health.Subsystems.{Cromwell, Sam, import scala.concurrent.{ExecutionContext, Future} -class MockStatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus]])(implicit system: ActorSystem, executionContext: ExecutionContext) extends - StatusService(checkStatus)(system, executionContext) { +class MockStatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus]])(implicit + system: ActorSystem, + executionContext: ExecutionContext +) extends StatusService(checkStatus)(system, executionContext) { override def status(): Future[StatusCheckResponse] = { val subsystemStatus: SubsystemStatus = SubsystemStatus(ok = true, None) @@ -16,4 +18,3 @@ class MockStatusService(checkStatus: () => Map[Subsystem, Future[SubsystemStatus Future.successful(StatusCheckResponse(ok = true, subsystems)) } } - diff --git a/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala index 5ad6a976204..e93acd51a2a 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/CromIamApiServiceSpec.scala @@ -4,7 +4,8 @@ import akka.event.NoLogging import akka.http.scaladsl.model.StatusCodes._ import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken, RawHeader} import akka.http.scaladsl.model.{ContentTypes, HttpEntity, HttpHeader} -import akka.http.scaladsl.server.MissingHeaderRejection +import akka.http.scaladsl.server.Route.seal +import akka.http.scaladsl.server.{AuthorizationFailedRejection, MissingHeaderRejection} import akka.http.scaladsl.testkit.ScalatestRouteTest import com.typesafe.config.Config import common.assertion.CromwellTimeoutSpec @@ -12,14 +13,23 @@ import cromiam.server.status.StatusService import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with CromIamApiService with ScalatestRouteTest { +class CromIamApiServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with CromIamApiService + with ScalatestRouteTest { override def testConfigSource = "akka.loglevel = DEBUG" val log = NoLogging - override def rootConfig: Config = throw new UnsupportedOperationException("This spec shouldn't need to access the real config") + override def rootConfig: Config = throw new UnsupportedOperationException( + "This spec shouldn't need to access the real config" + ) - override def configuration = throw new UnsupportedOperationException("This spec shouldn't need to access the real interface/port") + override def configuration = throw new UnsupportedOperationException( + "This spec shouldn't need to access the real interface/port" + ) override lazy val cromwellClient = new MockCromwellClient() override lazy val samClient = new MockSamClient() @@ -28,13 +38,15 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val version = "v1" val authorization = Authorization(OAuth2BearerToken("my-token")) - val badAuthHeaders: List[HttpHeader] = List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.unauthorizedUserCollectionStr)) - val goodAuthHeaders: List[HttpHeader] = List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.authorizedUserCollectionStr)) - + val badAuthHeaders: List[HttpHeader] = + List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.unauthorizedUserCollectionStr)) + val goodAuthHeaders: List[HttpHeader] = + List(authorization, RawHeader("OIDC_CLAIM_user_id", cromwellClient.authorizedUserCollectionStr)) behavior of "Status endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/status").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/status") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -42,7 +54,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -50,15 +63,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/status").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/status") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/status") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -71,10 +88,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Outputs endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/outputs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/outputs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -82,7 +99,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -90,15 +108,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/outputs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/outputs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/outputs") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -111,10 +133,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Metadata endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/metadata").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/metadata") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -122,7 +144,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -130,15 +153,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/metadata").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/metadata") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/metadata") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -151,10 +178,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Logs endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/logs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/logs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -162,7 +189,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -170,15 +198,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/logs").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/logs") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/logs") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -191,10 +223,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "GET Labels endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -202,7 +234,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 200 for authorized user who has collection associated with subworkflow" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -210,15 +243,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have view permissions" in { - Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -231,12 +268,13 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "PATCH Labels endpoint" it should "successfully forward request to Cromwell if nothing is untoward" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -247,7 +285,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val labels = """{"key-1":"foo","caas-collection-name":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe InternalServerError responseAs[String] shouldBe "Submitted labels contain the key caas-collection-name, which is not allowed\n" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -258,7 +298,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val labels = """"key-1":"foo"""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe InternalServerError responseAs[String] shouldBe "Labels must be a valid JSON object, received: \"key-1\":\"foo\"\n" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -268,9 +310,13 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "return 500 for authorized user who doesn't have collection associated with workflow" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels").withHeaders(goodAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/labels") + .withHeaders(goodAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } @@ -278,7 +324,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "return SamDenialException for user who doesn't have update permissions" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels").withHeaders(badAuthHeaders).withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.subworkflowId}/labels") + .withHeaders(badAuthHeaders) + .withEntity(labelEntity) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -288,12 +336,12 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "reject request if it doesn't contain OIDC_CLAIM_user_id in header" in { val labels = """{"key-1":"foo","key-2":"bar"}""" val labelEntity = HttpEntity(ContentTypes.`application/json`, labels) - Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels").withEntity(labelEntity) ~> allRoutes ~> check { + Patch(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/labels") + .withEntity(labelEntity) ~> allRoutes ~> check { rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") } } - behavior of "Backends endpoint" it should "successfully forward request to Cromwell if auth header is provided" in { Get(s"/api/workflows/$version/backends").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { @@ -303,16 +351,50 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - it should "reject request if it doesn't contain OIDC_CLAIM_user_id in header" in { + it should "reject request if it doesn't contain OIDC_CLAIM_user_id or token" in { Get(s"/api/workflows/$version/backends") ~> allRoutes ~> check { rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") } } + it should "return 403 when we request with a disabled user" in { + Get( + s"/api/workflows/$version/backends" + ).withHeaders( + List(Authorization(OAuth2BearerToken("my-token")), RawHeader("OIDC_CLAIM_user_id", "disabled@example.com")) + ) ~> allRoutes ~> check { + rejection shouldEqual AuthorizationFailedRejection + } + } + + it should "reject request if it contains a token and no OIDC_CLAIM_user_id in header" in { + Get( + s"/api/workflows/$version/backends" + ).withHeaders( + List(Authorization(OAuth2BearerToken("my-token"))) + ) ~> allRoutes ~> check { + rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") + } + } + + it should "return 404 when no auth token provided" in { + Get( + s"/api/workflows/$version/backends" + ).withHeaders( + List(RawHeader("OIDC_CLAIM_user_id", "enabled@example.com")) + // "[An] explicit call on the Route.seal method is needed in test code, but in your application code it is not necessary." + // https://doc.akka.io/docs/akka-http/current/routing-dsl/testkit.html#testing-sealed-routes + // https://doc.akka.io/docs/akka-http/current/routing-dsl/routes.html#sealing-a-route + ) ~> seal(allRoutes) ~> check { + responseAs[String] shouldEqual "The requested resource could not be found." + status shouldBe NotFound + } + } behavior of "ReleaseHold endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/releaseHold").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/releaseHold") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -320,15 +402,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/releaseHold").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/releaseHold") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have update permissions" in { - Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/releaseHold").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/releaseHold") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -341,10 +427,10 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "Abort endpoint" it should "return 200 for authorized user who has collection associated with root workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/abort").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.rootWorkflowIdWithCollection}/abort") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -352,15 +438,19 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who doesn't have collection associated with workflow" in { - Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/abort").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.workflowIdWithoutCollection}/abort") + .withHeaders(goodAuthHeaders) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have abort permissions" in { - Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/abort").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + Post(s"/api/workflows/$version/${cromwellClient.subworkflowId}/abort") + .withHeaders(badAuthHeaders) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -373,11 +463,13 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } } - behavior of "CallCacheDiff endpoint" it should "return 200 for authorized user who has collection associated with both workflows" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + goodAuthHeaders + ) ~> allRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Response from Cromwell" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -386,7 +478,9 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma it should "return BadRequest if request is malformed" in { val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + goodAuthHeaders + ) ~> allRoutes ~> check { status shouldBe BadRequest responseAs[String] shouldBe "Must supply both workflowA and workflowB to the /callcaching/diff endpoint" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -394,17 +488,25 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "return 500 for authorized user who has doesn't have collection associated with any one workflow" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.workflowIdWithoutCollection}&callB=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(goodAuthHeaders) ~> allRoutes ~> check { + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.workflowIdWithoutCollection}&callB=helloCall" + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + goodAuthHeaders + ) ~> allRoutes ~> check { status shouldBe InternalServerError - responseAs[String] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" + responseAs[ + String + ] shouldBe s"CromIAM unexpected error: java.lang.IllegalArgumentException: Workflow ${cromwellClient.workflowIdWithoutCollection} has no associated collection" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } it should "return SamDenialException for user who doesn't have read permissions" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" - Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders(badAuthHeaders) ~> allRoutes ~> check { + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" + Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams").withHeaders( + badAuthHeaders + ) ~> allRoutes ~> check { status shouldBe Forbidden responseAs[String] shouldBe "Access Denied" contentType should be(ContentTypes.`text/plain(UTF-8)`) @@ -412,7 +514,8 @@ class CromIamApiServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma } it should "reject request if it doesn't contain OIDC_CLAIM_user_id in header" in { - val callCacheDiffParams = s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" + val callCacheDiffParams = + s"workflowA=${cromwellClient.rootWorkflowIdWithCollection}&callA=helloCall&workflowB=${cromwellClient.anotherRootWorkflowIdWithCollection}&callB=helloCall" Get(s"/api/workflows/$version/callcaching/diff?$callCacheDiffParams") ~> allRoutes ~> check { rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") } diff --git a/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala index f6f1e3b75de..9d431f6af05 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/EngineRouteSupportSpec.scala @@ -9,8 +9,12 @@ import cromiam.server.status.{MockStatusService, StatusService} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - -class EngineRouteSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with EngineRouteSupport { +class EngineRouteSupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with EngineRouteSupport { override val cromwellClient = new MockCromwellClient() val samClient = new MockSamClient() override val statusService: StatusService = new MockStatusService(() => Map.empty) diff --git a/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala b/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala index 3fb9689e5cc..10f579aea8c 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/MockClients.scala @@ -16,10 +16,8 @@ import cromwell.api.model._ import scala.concurrent.ExecutionContextExecutor -class MockCromwellClient()(implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender)(system, ece, materializer) { +class MockCromwellClient()(implicit system: ActorSystem, ece: ExecutionContextExecutor, materializer: ActorMaterializer) + extends CromwellClient("http", "bar", 1, NoLogging, ActorRef.noSender)(system, ece, materializer) { val version = "v1" val unauthorizedUserCollectionStr: String = "987654321" @@ -56,7 +54,7 @@ class MockCromwellClient()(implicit system: ActorSystem, val womtoolRoutePath = s"/api/womtool/$version/describe" httpRequest.uri.path.toString match { - //version endpoint doesn't require authentication + // version endpoint doesn't require authentication case `versionRoutePath` => FailureResponseOrT.pure(HttpResponse(status = OK, entity = "Response from Cromwell")) // womtool endpoint requires authn which it gets for free from the proxy, does not care about authz @@ -68,18 +66,19 @@ class MockCromwellClient()(implicit system: ActorSystem, override def getRootWorkflow(workflowId: String, user: User, - cromIamRequest: HttpRequest): FailureResponseOrT[String] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[String] = workflowId match { case `subworkflowId` | `rootWorkflowIdWithCollection` => FailureResponseOrT.pure(rootWorkflowIdWithCollection) case `anotherRootWorkflowIdWithCollection` => FailureResponseOrT.pure(anotherRootWorkflowIdWithCollection) case _ => FailureResponseOrT.pure(workflowIdWithoutCollection) } - } override def collectionForWorkflow(workflowId: String, user: User, - cromIamRequest: HttpRequest): FailureResponseOrT[Collection] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Collection] = workflowId match { case `rootWorkflowIdWithCollection` | `anotherRootWorkflowIdWithCollection` => FailureResponseOrT.pure(userCollection) @@ -87,33 +86,32 @@ class MockCromwellClient()(implicit system: ActorSystem, val exception = new IllegalArgumentException(s"Workflow $workflowId has no associated collection") FailureResponseOrT.left(IO.raiseError[HttpResponse](exception)) } - } } /** * Overrides some values, but doesn't override methods. */ -class BaseMockSamClient(checkSubmitWhitelist: Boolean = true) - (implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends SamClient( - "http", - "bar", - 1, - checkSubmitWhitelist, - NoLogging, - ActorRef.noSender - )(system, ece, materializer) +class BaseMockSamClient(checkSubmitWhitelist: Boolean = true)(implicit + system: ActorSystem, + ece: ExecutionContextExecutor, + materializer: ActorMaterializer +) extends SamClient( + "http", + "bar", + 1, + checkSubmitWhitelist, + NoLogging, + ActorRef.noSender + )(system, ece, materializer) /** * Extends the base mock client with overriden methods. */ -class MockSamClient(checkSubmitWhitelist: Boolean = true) - (implicit system: ActorSystem, - ece: ExecutionContextExecutor, - materializer: ActorMaterializer) - extends BaseMockSamClient(checkSubmitWhitelist) { +class MockSamClient(checkSubmitWhitelist: Boolean = true)(implicit + system: ActorSystem, + ece: ExecutionContextExecutor, + materializer: ActorMaterializer +) extends BaseMockSamClient(checkSubmitWhitelist) { override def collectionsForUser(user: User, httpRequest: HttpRequest): FailureResponseOrT[List[Collection]] = { val userId = user.userId.value @@ -127,7 +125,8 @@ class MockSamClient(checkSubmitWhitelist: Boolean = true) override def requestSubmission(user: User, collection: Collection, - cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Unit] = collection match { case c if c.name.equalsIgnoreCase(UnauthorizedUserCollectionStr) => val exception = SamRegisterCollectionException(StatusCodes.BadRequest) @@ -135,19 +134,34 @@ class MockSamClient(checkSubmitWhitelist: Boolean = true) case c if c.name.equalsIgnoreCase(AuthorizedUserCollectionStr) => Monad[FailureResponseOrT].unit case _ => Monad[FailureResponseOrT].unit } - } - override def isSubmitWhitelistedSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + override def isSubmitWhitelistedSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = FailureResponseOrT.pure(!user.userId.value.equalsIgnoreCase(NotWhitelistedUser)) + + override def isUserEnabledSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = + if (user.userId.value == "enabled@example.com" || user.userId.value == MockSamClient.AuthorizedUserCollectionStr) + FailureResponseOrT.pure(true) + else if (user.userId.value == "disabled@example.com") + FailureResponseOrT.pure(false) + else + throw new Exception("Misconfigured test") + + override def isUserEnabledSam(user: User, cromIamRequest: HttpRequest): FailureResponseOrT[Boolean] = { + if (user.userId.value == "enabled@example.com" || user.userId.value == MockSamClient.AuthorizedUserCollectionStr) + FailureResponseOrT.pure(true) + else if (user.userId.value == "disabled@example.com") + FailureResponseOrT.pure(false) + else + throw new Exception("Misconfigured test") } override def requestAuth(authorizationRequest: CollectionAuthorizationRequest, - cromIamRequest: HttpRequest): FailureResponseOrT[Unit] = { + cromIamRequest: HttpRequest + ): FailureResponseOrT[Unit] = authorizationRequest.user.userId.value match { case AuthorizedUserCollectionStr => Monad[FailureResponseOrT].unit case _ => FailureResponseOrT.left(IO.raiseError[HttpResponse](new SamDenialException)) } - } } object MockSamClient { @@ -156,7 +170,6 @@ object MockSamClient { val NotWhitelistedUser: String = "ABC123" val UserCollectionList: List[Collection] = List(Collection("col1"), Collection("col2")) - def returnResponse[T](response: HttpResponse): FailureResponseOrT[T] = { + def returnResponse[T](response: HttpResponse): FailureResponseOrT[T] = FailureResponseOrT.left(IO.pure(response)) - } } diff --git a/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala index 216691c1c35..57dedbaefdf 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/QuerySupportSpec.scala @@ -11,8 +11,12 @@ import org.broadinstitute.dsde.workbench.model.WorkbenchUserId import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - -class QuerySupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with QuerySupport { +class QuerySupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with QuerySupport { override val cromwellClient = new MockCromwellClient() override val samClient = new MockSamClient() override val log: LoggingAdapter = NoLogging @@ -27,7 +31,8 @@ class QuerySupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher val getQuery = s"$queryPath?status=Submitted&label=foo:bar&label=foo:baz" val badGetQuery = s"$queryPath?status=Submitted&labelor=foo:bar&label=foo:baz" - val goodPostEntity = HttpEntity(ContentTypes.`application/json`, + val goodPostEntity = HttpEntity( + ContentTypes.`application/json`, """|[ | { | "start": "2015-11-01T00:00:00-04:00" @@ -50,7 +55,8 @@ class QuerySupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher |] |""".stripMargin ) - val badPostEntity = HttpEntity(ContentTypes.`application/json`, + val badPostEntity = HttpEntity( + ContentTypes.`application/json`, """|[ | { | "start": "2015-11-01T00:00:00-04:00" diff --git a/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala index c4a424747e6..bfbd38075f5 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/SubmissionSupportSpec.scala @@ -12,8 +12,12 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ - -class SubmissionSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SubmissionSupport { +class SubmissionSupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with SubmissionSupport { override val cromwellClient = new MockCromwellClient() override val samClient = new MockSamClient() override val log: LoggingAdapter = NoLogging @@ -56,7 +60,6 @@ class SubmissionSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val workflowSource = Multipart.FormData.BodyPart("workflowSource", HttpEntity(helloWorldWdl)) val formData = Multipart.FormData(workflowSource).toEntity() - "Submit endpoint" should "forward the request to Cromwell for authorized SAM user" in { Post(submitPath).withHeaders(goodAuthHeaders).withEntity(formData) ~> submitRoute ~> check { status shouldEqual StatusCodes.OK diff --git a/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala index c0277fd92c0..32a7511c6de 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/SwaggerServiceSpec.scala @@ -12,15 +12,20 @@ import org.scalatest.prop.TableDrivenPropertyChecks import org.yaml.snakeyaml.constructor.Constructor import org.yaml.snakeyaml.error.YAMLException import org.yaml.snakeyaml.nodes.MappingNode -import org.yaml.snakeyaml.{Yaml => SnakeYaml} +import org.yaml.snakeyaml.{LoaderOptions, Yaml => SnakeYaml} import scala.jdk.CollectionConverters._ - -class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with SwaggerService with ScalatestRouteTest with Matchers - with TableDrivenPropertyChecks { +class SwaggerServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with SwaggerService + with ScalatestRouteTest + with Matchers + with TableDrivenPropertyChecks { def actorRefFactory = system override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig("clientId", "realm", "appName") + val yamlLoaderOptions = new LoaderOptions behavior of "SwaggerService" @@ -32,7 +37,8 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg contentType should be(ContentTypes.`application/octet-stream`) val body = responseAs[String] - val yaml = new SnakeYaml(new UniqueKeyConstructor()).loadAs(body, classOf[java.util.Map[String, AnyRef]]) + val yaml = new SnakeYaml(new UniqueKeyConstructor(new LoaderOptions)) + .loadAs(body, classOf[java.util.Map[String, AnyRef]]) yaml.get("swagger") should be("2.0") } @@ -61,27 +67,42 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg resultWithInfo.getSwagger.getDefinitions.asScala foreach { // If no properties, `getProperties` returns `null` instead of an empty map - case (defKey, defVal) => Option(defVal.getProperties).map(_.asScala).getOrElse(Map.empty) foreach { - /* + case (defKey, defVal) => + Option(defVal.getProperties).map(_.asScala).getOrElse(Map.empty) foreach { + /* Two against one. Swagger parser implementation lets a RefProperty have descriptions. http://swagger.io/specification/#referenceObject & http://editor.swagger.io both say it's ref ONLY! - */ - case (propKey, propVal: RefProperty) => - withClue(s"RefProperty $defKey.$propKey has a description: ") { - propVal.getDescription should be(null) - } - case _ => /* ignore */ - } + */ + case (propKey, propVal: RefProperty) => + withClue(s"RefProperty $defKey.$propKey has a description: ") { + propVal.getDescription should be(null) + } + case _ => /* ignore */ + } } } } it should "return status OK when getting OPTIONS on paths" in { - val pathExamples = Table("path", "/", "/swagger", "/swagger/cromwell.yaml", "/swagger/index.html", "/api", - "/api/workflows/", "/api/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status", - "/api/workflows/v1/validate", "/workflows", "/workflows/v1", "/workflows/v1/outputs", "/workflows/v1/status", - "/workflows/v1/validate") + val pathExamples = Table( + "path", + "/", + "/swagger", + "/swagger/cromwell.yaml", + "/swagger/index.html", + "/api", + "/api/workflows/", + "/api/workflows/v1", + "/workflows/v1/outputs", + "/workflows/v1/status", + "/api/workflows/v1/validate", + "/workflows", + "/workflows/v1", + "/workflows/v1/outputs", + "/workflows/v1/status", + "/workflows/v1/validate" + ) forAll(pathExamples) { path => Options(path) ~> @@ -109,7 +130,7 @@ class SwaggerServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Swagg * Adapted from: * https://bitbucket.org/asomov/snakeyaml/src/e9cd9f5e8d76c61eb983e29b3dc039c1fac9c393/src/test/java/org/yaml/snakeyaml/issues/issue139/UniqueKeyTest.java?fileviewer=file-view-default#UniqueKeyTest.java-43:62 */ -class UniqueKeyConstructor extends Constructor { +class UniqueKeyConstructor(val loaderOptions: LoaderOptions) extends Constructor(loaderOptions) { import java.util.{Map => JMap} diff --git a/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala index 1d0103a09fd..b39466ec4f0 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/SwaggerUiHttpServiceSpec.scala @@ -10,19 +10,38 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks - -trait SwaggerUiHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with SwaggerUiHttpService { +trait SwaggerUiHttpServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with SwaggerUiHttpService { def actorRefFactory = system } -trait SwaggerResourceHttpServiceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with ScalatestRouteTest with -TableDrivenPropertyChecks with SwaggerResourceHttpService { - - val testPathsForOptions = Table("endpoint", "/", "/swagger", "/swagger/index.html", "/api", "/api/example", - "/api/example?with=param", "/api/example/path") +trait SwaggerResourceHttpServiceSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with ScalatestRouteTest + with TableDrivenPropertyChecks + with SwaggerResourceHttpService { + + val testPathsForOptions = Table("endpoint", + "/", + "/swagger", + "/swagger/index.html", + "/api", + "/api/example", + "/api/example?with=param", + "/api/example/path" + ) } -trait SwaggerUiResourceHttpServiceSpec extends SwaggerUiHttpServiceSpec with SwaggerResourceHttpServiceSpec with SwaggerUiResourceHttpService +trait SwaggerUiResourceHttpServiceSpec + extends SwaggerUiHttpServiceSpec + with SwaggerResourceHttpServiceSpec + with SwaggerUiResourceHttpService object SwaggerUiHttpServiceSpec { // TODO: Re-common-ize swagger out of cromwell's engine and reuse. @@ -40,7 +59,7 @@ class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec { behavior of "SwaggerUiHttpService" override protected def rewriteSwaggerIndex(data: String): String = - // Replace same magic string used in SwaggerUiResourceHttpService.rewriteSwaggerIndex + // Replace same magic string used in SwaggerUiResourceHttpService.rewriteSwaggerIndex data.replace("window.ui = ui", "replaced-client-id") it should "redirect /swagger to /" in { @@ -80,7 +99,9 @@ class BasicSwaggerUiHttpServiceSpec extends SwaggerUiHttpServiceSpec { } override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig( - clientId = "test-client-id", realm = "test-realm", appName = "test-appname" + clientId = "test-client-id", + realm = "test-realm", + appName = "test-appname" ) } @@ -195,7 +216,6 @@ class YamlSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceS } } - class JsonSwaggerUiResourceHttpServiceSpec extends SwaggerUiResourceHttpServiceSpec { override def oauthConfig: SwaggerOauthConfig = SwaggerOauthConfig("clientId", "realm", "appName") diff --git a/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala b/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala index 7ba1c495f23..a229f2db255 100644 --- a/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala +++ b/CromIAM/src/test/scala/cromiam/webservice/WomtoolRouteSupportSpec.scala @@ -2,24 +2,80 @@ package cromiam.webservice import akka.http.scaladsl.model.ContentTypes import akka.http.scaladsl.model.StatusCodes._ +import akka.http.scaladsl.model.headers.{Authorization, OAuth2BearerToken, RawHeader} +import akka.http.scaladsl.server.Route.seal +import akka.http.scaladsl.server.{AuthorizationFailedRejection, MissingHeaderRejection} import akka.http.scaladsl.testkit.ScalatestRouteTest import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - -class WomtoolRouteSupportSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with WomtoolRouteSupport with ScalatestRouteTest { +class WomtoolRouteSupportSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with WomtoolRouteSupport + with ScalatestRouteTest { override lazy val cromwellClient = new MockCromwellClient() + override lazy val samClient = new MockSamClient() behavior of "Womtool endpoint routes" it should "return 200 when we request to the right path" in { - Post(s"/api/womtool/v1/describe") ~> womtoolRoutes ~> check { + Post( + s"/api/womtool/v1/describe" + ).withHeaders( + List(Authorization(OAuth2BearerToken("my-token")), RawHeader("OIDC_CLAIM_user_id", "enabled@example.com")) + ) ~> womtoolRoutes ~> check { status shouldBe OK responseAs[String] shouldBe "Hey there, workflow describer" contentType should be(ContentTypes.`text/plain(UTF-8)`) } } + it should "return 403 when we request with a disabled user" in { + Post( + s"/api/womtool/v1/describe" + ).withHeaders( + List(Authorization(OAuth2BearerToken("my-token")), RawHeader("OIDC_CLAIM_user_id", "disabled@example.com")) + ) ~> womtoolRoutes ~> check { + rejection shouldEqual AuthorizationFailedRejection + } + } + + it should "bail out with no user ID" in { + Post( + s"/api/womtool/v1/describe" + ).withHeaders( + List(Authorization(OAuth2BearerToken("my-token"))) + ) ~> womtoolRoutes ~> check { + rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") + } + } + + it should "return 404 when no auth token provided" in { + Post( + s"/api/womtool/v1/describe" + ).withHeaders( + List(RawHeader("OIDC_CLAIM_user_id", "enabled@example.com")) + // "[An] explicit call on the Route.seal method is needed in test code, but in your application code it is not necessary." + // https://doc.akka.io/docs/akka-http/current/routing-dsl/testkit.html#testing-sealed-routes + // https://doc.akka.io/docs/akka-http/current/routing-dsl/routes.html#sealing-a-route + ) ~> seal(womtoolRoutes) ~> check { + responseAs[String] shouldEqual "The requested resource could not be found." + status shouldBe NotFound + } + } + + it should "bail out with no headers" in { + Post( + s"/api/womtool/v1/describe" + ).withHeaders( + List.empty + ) ~> womtoolRoutes ~> check { + rejection shouldEqual MissingHeaderRejection("OIDC_CLAIM_user_id") + } + } + } diff --git a/CromwellRefdiskManifestCreator/pom.xml b/CromwellRefdiskManifestCreator/pom.xml index 987aad421be..829f64b5fd3 100644 --- a/CromwellRefdiskManifestCreator/pom.xml +++ b/CromwellRefdiskManifestCreator/pom.xml @@ -48,7 +48,7 @@ com.fasterxml.jackson.core jackson-databind - 2.13.2.2 + 2.13.4.1 org.apache.logging.log4j diff --git a/README.md b/README.md index fac1541cf8e..ec997ab4f03 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,16 @@ -[![Build Status](https://travis-ci.com/broadinstitute/cromwell.svg?branch=develop)](https://travis-ci.com/broadinstitute/cromwell?branch=develop) [![codecov](https://codecov.io/gh/broadinstitute/cromwell/branch/develop/graph/badge.svg)](https://codecov.io/gh/broadinstitute/cromwell) -## Welcome to Cromwell +## Welcome to the "AWS-friendly" Cromwell + +The AWS-friendly Cromwell is an optimized fork of the main cromwell release. We try to keep it up-to-date with new releases, while keeping our additions functional. + +* Information regarding AWS features can be found [here](supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/README.md) +* Information regarding deployment can be found [here](supportedBackends/aws/src/main/scala/cromwell/backend/impl/aws/DEPLOY.md) + +Contact: +* henrique [at] loka [dot] com +* geert [dot] vandeweyer [at] uza [dot] be +* Join the #AWS channel at the [Cromwell Slack workspace](https://join.slack.com/t/cromwellhq/shared_invite/zt-dxmmrtye-JHxwKE53rfKE_ZWdOHIB4g). Cromwell is an open-source Workflow Management System for bioinformatics. Licensing is [BSD 3-Clause](LICENSE.txt). diff --git a/azure-blob-nio/README.md b/azure-blob-nio/README.md new file mode 100644 index 00000000000..ad6c553eabf --- /dev/null +++ b/azure-blob-nio/README.md @@ -0,0 +1,5 @@ +# Azure Storage Blob NIO FileSystemProvider + +[This is a copy of the NIO Filesystem implementation version 12.0.0-beta.19](https://github.com/Azure/azure-sdk-for-java/tree/2490e1e19e8531fe0a6378f40e299e7ec64cf3aa/sdk/storage/azure-storage-blob-nio) + +For more information on the initial design and commit history see the Azure SDK repository linked above. Changes to this repo were necessary to support some of the specific needs Cromwell as an App on Azure has as a system in Terra. This is something that has some precedent as it has been done for other filesystems in the past. diff --git a/azure-blob-nio/assets.json b/azure-blob-nio/assets.json new file mode 100644 index 00000000000..c262f7ebafc --- /dev/null +++ b/azure-blob-nio/assets.json @@ -0,0 +1,6 @@ +{ + "AssetsRepo": "Azure/azure-sdk-assets", + "AssetsRepoPrefixPath": "java", + "TagPrefix": "java/storage/azure-storage-blob-nio", + "Tag": "java/storage/azure-storage-blob-nio_b2a0ce219e" +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBasicFileAttributeView.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBasicFileAttributeView.java new file mode 100644 index 00000000000..43744893ccb --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBasicFileAttributeView.java @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; + +/** + * Provides support for basic file attributes. + *

+ * The operations supported by this view and the attributes it reads are a strict subset of + * {@link AzureBlobFileAttributeView} and has the same network behavior. Therefore, while this type is offered for + * compliance with the NIO spec, {@link AzureBlobFileAttributeView} is generally preferred. + *

+ * {@link #setTimes(FileTime, FileTime, FileTime)} is not supported. + */ +public final class AzureBasicFileAttributeView implements BasicFileAttributeView { + private static final ClientLogger LOGGER = new ClientLogger(AzureBasicFileAttributeView.class); + + static final String NAME = "azureBasic"; + + private final Path path; + + AzureBasicFileAttributeView(Path path) { + this.path = path; + } + + /** + * Returns the name of the attribute view: {@code "azureBasic"} + * + * @return the name of the attribute view: {@code "azureBasic"} + */ + @Override + public String name() { + return NAME; + } + + /** + * Reads the basic file attributes as a bulk operation. + *

+ * All file attributes are read as an atomic operation with respect to other file system operations. + * + * @return {@link AzureBasicFileAttributes} + */ + @Override + public AzureBasicFileAttributes readAttributes() throws IOException { + AzurePath.ensureFileSystemOpen(path); + return new AzureBasicFileAttributes(path); + } + + /** + * Unsupported. + * + * @param lastModifiedTime the new last modified time, or null to not change the value + * @param lastAccessTime the last access time, or null to not change the value + * @param createTime the file's create time, or null to not change the value + * @throws UnsupportedOperationException Operation not supported. + * @throws IOException never + */ + @Override + public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) throws IOException { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException()); + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBasicFileAttributes.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBasicFileAttributes.java new file mode 100644 index 00000000000..d1ab6d28562 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBasicFileAttributes.java @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileTime; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Provides support for basic file attributes. + *

+ * The properties available on this type are a strict subset of {@link AzureBlobFileAttributes}, and the two types have + * the same network behavior. Therefore, while this type is offered for compliance with the NIO spec, + * {@link AzureBlobFileAttributes} is generally preferred. + *

+ * Some attributes are not supported. Refer to the javadocs on each method for more information. + *

+ * If the target file is a virtual directory, most attributes will be set to null. + */ +public final class AzureBasicFileAttributes implements BasicFileAttributes { + // For verifying parameters on FileSystemProvider.readAttributes + static final Set ATTRIBUTE_STRINGS; + static { + Set set = new HashSet<>(); + set.add("lastModifiedTime"); + set.add("isRegularFile"); + set.add("isDirectory"); + set.add("isVirtualDirectory"); + set.add("isSymbolicLink"); + set.add("isOther"); + set.add("size"); + set.add("creationTime"); + ATTRIBUTE_STRINGS = Collections.unmodifiableSet(set); + } + + private final AzureBlobFileAttributes internalAttributes; + + /* + In order to support Files.exist() and other methods like Files.walkFileTree() which depend on it, we have had to add + support for virtual directories. This is not ideal as customers will have to now perform null checks when inspecting + attributes (or at least check if it is a virtual directory before inspecting properties). It also incurs extra + network requests as we have to call a checkDirectoryExists() after receiving the initial 404. This is two + additional network requests, though they only happen in the case when a file doesn't exist or is virtual, so it + shouldn't happen in the majority of api calls. + */ + AzureBasicFileAttributes(Path path) throws IOException { + this.internalAttributes = new AzureBlobFileAttributes(path); + } + + /** + * Returns the time of last modification or null if this is a virtual directory. + * + * @return the time of last modification or null if this is a virtual directory + */ + @Override + public FileTime lastModifiedTime() { + return this.internalAttributes.lastModifiedTime(); + } + + /** + * Returns the time of last modification or null if this is a virtual directory + *

+ * Last access time is not supported by the blob service. In this case, it is typical for implementations to return + * the {@link #lastModifiedTime()}. + * + * @return the time of last modification or null if this is a virtual directory + */ + @Override + public FileTime lastAccessTime() { + return this.internalAttributes.lastAccessTime(); + } + + /** + * Returns the creation time. The creation time is the time that the file was created. Returns null if this is a + * virtual directory. + * + * @return The creation time or null if this is a virtual directory + */ + @Override + public FileTime creationTime() { + return this.internalAttributes.creationTime(); + } + + /** + * Tells whether the file is a regular file with opaque content. + * + * @return whether the file is a regular file. + */ + @Override + public boolean isRegularFile() { + return this.internalAttributes.isRegularFile(); + } + + /** + * Tells whether the file is a directory. + *

+ * Will only return true if the directory is a concrete directory. See + * {@link AzureFileSystemProvider#createDirectory(Path, FileAttribute[])} for more information on virtual and + * concrete directories. + * + * @return whether the file is a directory + */ + @Override + public boolean isDirectory() { + return this.internalAttributes.isDirectory(); + } + + /** + * Tells whether the file is a virtual directory. + *

+ * See {@link AzureFileSystemProvider#createDirectory(Path, FileAttribute[])} for more information on virtual and + * concrete directories. + * + * @return whether the file is a virtual directory + */ + public boolean isVirtualDirectory() { + return this.internalAttributes.isVirtualDirectory(); + } + + /** + * Tells whether the file is a symbolic link. + * + * @return false. Symbolic links are not supported. + */ + @Override + public boolean isSymbolicLink() { + return this.internalAttributes.isSymbolicLink(); + } + + /** + * Tells whether the file is something other than a regular file, directory, or symbolic link. + * + * @return false. No other object types are supported. + */ + @Override + public boolean isOther() { + return this.internalAttributes.isOther(); + } + + /** + * Returns the size of the file (in bytes). + * + * @return the size of the file + */ + @Override + public long size() { + return this.internalAttributes.size(); + } + + /** + * Returns the url of the resource. + * + * @return The file key, which is the url. + */ + @Override + public Object fileKey() { + return this.internalAttributes.fileKey(); + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBlobFileAttributeView.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBlobFileAttributeView.java new file mode 100644 index 00000000000..d9366e22417 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBlobFileAttributeView.java @@ -0,0 +1,157 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.models.AccessTier; +import com.azure.storage.blob.models.BlobHttpHeaders; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.specialized.BlobClientBase; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileTime; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Consumer; + +/** + * A file attribute view that provides a view of attributes specific to files stored as blobs in Azure Storage. + *

+ * All attributes are retrieved from the file system as a bulk operation. + *

+ * {@link #setTimes(FileTime, FileTime, FileTime)} is not supported. + */ +public final class AzureBlobFileAttributeView implements BasicFileAttributeView { + private static final ClientLogger LOGGER = new ClientLogger(AzureBlobFileAttributeView.class); + + static final String ATTR_CONSUMER_ERROR = "Exception thrown by attribute consumer"; + static final String NAME = "azureBlob"; + + private final Path path; + + AzureBlobFileAttributeView(Path path) { + this.path = path; + } + + @SuppressWarnings("unchecked") + static Map> setAttributeConsumers(AzureBlobFileAttributeView view) { + Map> map = new HashMap<>(); + map.put("blobHttpHeaders", obj -> { + try { + view.setBlobHttpHeaders((BlobHttpHeaders) obj); + } catch (IOException e) { + throw LoggingUtility.logError(LOGGER, new UncheckedIOException(ATTR_CONSUMER_ERROR, e)); + } + }); + map.put("metadata", obj -> { + try { + Map m = (Map) obj; + if (m == null) { + throw LoggingUtility.logError(LOGGER, new ClassCastException()); + } + view.setMetadata(m); + } catch (IOException e) { + throw LoggingUtility.logError(LOGGER, new UncheckedIOException(ATTR_CONSUMER_ERROR, e)); + } + }); + map.put("tier", obj -> { + try { + view.setTier((AccessTier) obj); + } catch (IOException e) { + throw LoggingUtility.logError(LOGGER, new UncheckedIOException(ATTR_CONSUMER_ERROR, e)); + } + }); + + return map; + } + + /** + * Returns the name of the attribute view: {@code "azureBlob"} + * + * @return the name of the attribute view: {@code "azureBlob"} + */ + @Override + public String name() { + return NAME; + } + + /** + * Reads the file attributes as a bulk operation. + *

+ * All file attributes are read as an atomic operation with respect to other file system operations. A fresh copy is + * retrieved every time this method is called. + * @return {@link AzureBlobFileAttributes} + * @throws IOException if an IOException occurs. + */ + @Override + public AzureBlobFileAttributes readAttributes() throws IOException { + AzurePath.ensureFileSystemOpen(path); + return new AzureBlobFileAttributes(path); + } + + /** + * Sets the {@link BlobHttpHeaders} as an atomic operation. + *

+ * See {@link BlobClientBase#setHttpHeaders(BlobHttpHeaders)} for more information. + * @param headers {@link BlobHttpHeaders} + * @throws IOException if an IOException occurs. + */ + public void setBlobHttpHeaders(BlobHttpHeaders headers) throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + new AzureResource(this.path).getBlobClient().setHttpHeaders(headers); + } catch (BlobStorageException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + /** + * Sets the metadata as an atomic operation. + *

+ * See {@link BlobClientBase#setMetadata(Map)} for more information. + * @param metadata The metadata to associate with the blob + * @throws IOException if an IOException occurs. + */ + public void setMetadata(Map metadata) throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + new AzureResource(this.path).getBlobClient().setMetadata(metadata); + } catch (BlobStorageException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + /** + * Sets the {@link AccessTier} on the file. + *

+ * See {@link BlobClientBase#setAccessTier(AccessTier)} for more information. + * @param tier {@link AccessTier} + * @throws IOException if an IOException occurs. + */ + public void setTier(AccessTier tier) throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + new AzureResource(this.path).getBlobClient().setAccessTier(tier); + } catch (BlobStorageException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + /** + * Unsupported. + * + * @param lastModifiedTime the new last modified time, or null to not change the value + * @param lastAccessTime the last access time, or null to not change the value + * @param createTime the file's create time, or null to not change the value + * @throws UnsupportedOperationException Operation not supported. + * @throws IOException never + */ + @Override + public void setTimes(FileTime lastModifiedTime, FileTime lastAccessTime, FileTime createTime) throws IOException { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException()); + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBlobFileAttributes.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBlobFileAttributes.java new file mode 100644 index 00000000000..c73d062e117 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureBlobFileAttributes.java @@ -0,0 +1,369 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileTime; +import java.time.OffsetDateTime; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.function.Supplier; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.models.AccessTier; +import com.azure.storage.blob.models.ArchiveStatus; +import com.azure.storage.blob.models.BlobHttpHeaders; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.BlobType; +import com.azure.storage.blob.models.CopyStatusType; + +/** + * Provides support for attributes associated with a file stored as a blob in Azure Storage. + *

+ * Some of the attributes inherited from {@link BasicFileAttributes} are not supported. See the docs on each method for + * more information. + *

+ * If the target file is a virtual directory, most attributes will be set to null. + */ +public final class AzureBlobFileAttributes implements BasicFileAttributes { + /* + Some blob properties do not have getters as they do not make sense in the context of nio. These properties are: + - incremental snapshot related properties (only for page blobs) + - lease related properties (leases not currently supported) + - sequence number (only for page blobs) + - encryption key sha256 (cpk not supported) + - committed block count (only for append blobs) + */ + + private static final ClientLogger LOGGER = new ClientLogger(AzureBlobFileAttributes.class); + + private final BlobProperties properties; + private final AzureResource resource; + private final boolean isVirtualDirectory; + + AzureBlobFileAttributes(Path path) throws IOException { + this.resource = new AzureResource(path); + BlobProperties props = null; + try { + props = resource.getBlobClient().getProperties(); + } catch (BlobStorageException e) { + if (e.getStatusCode() == 404 && this.resource.checkVirtualDirectoryExists()) { + this.isVirtualDirectory = true; + this.properties = null; + return; + } else { + throw LoggingUtility.logError(LOGGER, new IOException("Path: " + path.toString(), e)); + } + } + this.properties = props; + this.isVirtualDirectory = false; + } + + static Map> getAttributeSuppliers(AzureBlobFileAttributes attributes) { + Map> map = new HashMap<>(); + map.put("creationTime", attributes::creationTime); + map.put("lastModifiedTime", attributes::lastModifiedTime); + map.put("eTag", attributes::eTag); + map.put("blobHttpHeaders", attributes::blobHttpHeaders); + map.put("blobType", attributes::blobType); + map.put("copyId", attributes::copyId); + map.put("copyStatus", attributes::copyStatus); + map.put("copySource", attributes::copySource); + map.put("copyProgress", attributes::copyProgress); + map.put("copyCompletionTime", attributes::copyCompletionTime); + map.put("copyStatusDescription", attributes::copyStatusDescription); + map.put("isServerEncrypted", attributes::isServerEncrypted); + map.put("accessTier", attributes::accessTier); + map.put("isAccessTierInferred", attributes::isAccessTierInferred); + map.put("archiveStatus", attributes::archiveStatus); + map.put("accessTierChangeTime", attributes::accessTierChangeTime); + map.put("metadata", attributes::metadata); + map.put("isRegularFile", attributes::isRegularFile); + map.put("isDirectory", attributes::isDirectory); + map.put("isVirtualDirectory", attributes::isVirtualDirectory); + map.put("isSymbolicLink", attributes::isSymbolicLink); + map.put("isOther", attributes::isOther); + map.put("size", attributes::size); + return map; + } + + /** + * Returns the creation time. The creation time is the time that the file was created. Returns null if this is a + * virtual directory. + * + * @return The creation time or null if this is a virtual directory + */ + @Override + public FileTime creationTime() { + return !this.isVirtualDirectory ? FileTime.from(this.properties.getCreationTime().toInstant()) : null; + } + + /** + * Returns the time of last modification. Returns null if this is a virtual directory + * + * @return the time of last modification or null if this is a virtual directory + */ + @Override + public FileTime lastModifiedTime() { + return !this.isVirtualDirectory ? FileTime.from(this.properties.getLastModified().toInstant()) : null; + } + + /** + * Returns the eTag of the blob or null if this is a virtual directory + * + * @return the eTag of the blob or null if this is a virtual directory + */ + public String eTag() { + return !this.isVirtualDirectory ? this.properties.getETag() : null; + } + + /** + * Returns the {@link BlobHttpHeaders} of the blob or null if this is a virtual directory. + * + * @return {@link BlobHttpHeaders} or null if this is a virtual directory + */ + public BlobHttpHeaders blobHttpHeaders() { + if (this.isVirtualDirectory) { + return null; + } + /* + We return these all as one value, so it's consistent with the way of setting, especially the setAttribute method + that accepts a string argument for the name of the property. Returning them individually would mean we have to + support setting them individually as well, which is not possible due to service constraints. + */ + return new BlobHttpHeaders() + .setContentType(this.properties.getContentType()) + .setContentLanguage(this.properties.getContentLanguage()) + .setContentMd5(this.properties.getContentMd5()) + .setContentDisposition(this.properties.getContentDisposition()) + .setContentEncoding(this.properties.getContentEncoding()) + .setCacheControl(this.properties.getCacheControl()); + } + + /** + * Returns the type of the blob or null if this is a virtual directory + * + * @return the type of the blob or null if this is a virtual directory + */ + public BlobType blobType() { + return !this.isVirtualDirectory ? this.properties.getBlobType() : null; + } + + /** + * Returns the identifier of the last copy operation. If this blob hasn't been the target of a copy operation or has + * been modified since this won't be set. Returns null if this is a virtual directory + * + * @return the identifier of the last copy operation or null if this is a virtual directory + */ + public String copyId() { + return !this.isVirtualDirectory ? this.properties.getCopyId() : null; + } + + /** + * Returns the status of the last copy operation. If this blob hasn't been the target of a copy operation or has + * been modified since this won't be set. Returns null if this is a virtual directory + * + * @return the status of the last copy operation or null if this is a virtual directory + */ + public CopyStatusType copyStatus() { + return !this.isVirtualDirectory ? this.properties.getCopyStatus() : null; + } + + /** + * Returns the source blob URL from the last copy operation. If this blob hasn't been the target of a copy operation + * or has been modified since this won't be set. Returns null if this is a virtual directory + * + * @return the source blob URL from the last copy operation or null if this is a virtual directory + */ + public String copySource() { + return !this.isVirtualDirectory ? this.properties.getCopySource() : null; + } + + /** + * Returns the number of bytes copied and total bytes in the source from the last copy operation (bytes copied/total + * bytes). If this blob hasn't been the target of a copy operation or has been modified since this won't be set. + * Returns null if this is a virtual directory + * + * @return the number of bytes copied and total bytes in the source from the last copy operation null if this is a + * virtual directory + */ + public String copyProgress() { + return !this.isVirtualDirectory ? this.properties.getCopyProgress() : null; + } + + /** + * Returns the completion time of the last copy operation. If this blob hasn't been the target of a copy operation + * or has been modified since this won't be set. Returns null if this is a virtual directory. + * + * @return the completion time of the last copy operation or null if this is a virtual directory + */ + public OffsetDateTime copyCompletionTime() { + return !this.isVirtualDirectory ? this.properties.getCopyCompletionTime() : null; + } + + /** + * Returns the description of the last copy failure, this is set when the {@link #copyStatus() getCopyStatus} is + * {@link CopyStatusType#FAILED failed} or {@link CopyStatusType#ABORTED aborted}. If this blob hasn't been the + * target of a copy operation or has been modified since this won't be set. Returns null if this is a virtual + * directory. + * + * @return the description of the last copy failure or null if this is a virtual directory + */ + public String copyStatusDescription() { + return !this.isVirtualDirectory ? this.properties.getCopyStatusDescription() : null; + } + + /** + * Returns the status of the blob being encrypted on the server or null if this is a virtual directory. + * + * @return the status of the blob being encrypted on the server or null if this is a virtual directory + */ + public Boolean isServerEncrypted() { + return !this.isVirtualDirectory ? this.properties.isServerEncrypted() : null; + } + + /** + * Returns the tier of the blob. This is only set for Page blobs on a premium storage account or for Block blobs on + * blob storage or general purpose V2 account. Returns null if this is a virtual directory. + * + * @return the tier of the blob or null if this is a virtual directory + */ + public AccessTier accessTier() { + return !this.isVirtualDirectory ? this.properties.getAccessTier() : null; + } + + /** + * Returns the status of the tier being inferred for the blob. This is only set for Page blobs on a premium storage + * account or for Block blobs on blob storage or general purpose V2 account. Returns null if this is a virtual + * directory. + * + * @return the status of the tier being inferred for the blob or null if this is a virtual directory + */ + public Boolean isAccessTierInferred() { + return !this.isVirtualDirectory ? this.properties.isAccessTierInferred() : null; + } + + /** + * Returns the archive status of the blob. This is only for blobs on a blob storage and general purpose v2 account. + * Returns null if this is a virtual directory. + * + * @return the archive status of the blob or null if this is a virtual directory + */ + public ArchiveStatus archiveStatus() { + return !this.isVirtualDirectory ? this.properties.getArchiveStatus() : null; + } + + /** + * Returns the time when the access tier for the blob was last changed or null if this is a virtual directory. + * + * @return the time when the access tier for the blob was last changed or null if this is a virtual directory + */ + public OffsetDateTime accessTierChangeTime() { + return !this.isVirtualDirectory ? this.properties.getAccessTierChangeTime() : null; + } + + /** + * Returns the metadata associated with this blob or null if this is a virtual directory. + * + * @return the metadata associated with this blob or null if this is a virtual directory + */ + public Map metadata() { + return !this.isVirtualDirectory ? Collections.unmodifiableMap(this.properties.getMetadata()) : null; + } + + /** + * Returns the time of last modification or null if this is a virtual directory. + *

+ * Last access time is not supported by the blob service. In this case, it is typical for implementations to return + * the {@link #lastModifiedTime()}. + * + * @return the time of last modification or null if this is a virtual directory + */ + @Override + public FileTime lastAccessTime() { + return !this.isVirtualDirectory ? FileTime.from(this.properties.getLastAccessedTime().toInstant()) : null; + } + + /** + * Tells whether the file is a regular file with opaque content. + * + * @return whether the file is a regular file. + */ + @Override + public boolean isRegularFile() { + return !this.isVirtualDirectory + && !this.properties.getMetadata().getOrDefault(AzureResource.DIR_METADATA_MARKER, "false").equals("true"); + } + + /** + * Tells whether the file is a directory. + *

+ * Will return true if the directory is a concrete or virtual directory. See + * {@link AzureFileSystemProvider#createDirectory(Path, FileAttribute[])} for more information on virtual and + * concrete directories. + * + * @return whether the file is a directory + */ + @Override + public boolean isDirectory() { + return !this.isRegularFile(); + } + + /** + * Tells whether the file is a virtual directory. + *

+ * See {@link AzureFileSystemProvider#createDirectory(Path, FileAttribute[])} for more information on virtual and + * concrete directories. + * + * @return whether the file is a virtual directory + */ + public boolean isVirtualDirectory() { + return this.isVirtualDirectory; + } + + /** + * Tells whether the file is a symbolic link. + * + * @return false. Symbolic links are not supported. + */ + @Override + public boolean isSymbolicLink() { + return false; + } + + /** + * Tells whether the file is something other than a regular file, directory, or symbolic link. + * + * @return false. No other object types are supported. + */ + @Override + public boolean isOther() { + return false; + } + + /** + * Returns the size of the file (in bytes). + * + * @return the size of the file + */ + @Override + public long size() { + return !this.isVirtualDirectory ? properties.getBlobSize() : 0; + } + + /** + * Returns the url of the resource. + * + * @return The file key, which is the url. + */ + @Override + public Object fileKey() { + return resource.getBlobClient().getBlobUrl(); + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureDirectoryStream.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureDirectoryStream.java new file mode 100644 index 00000000000..817121e958e --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureDirectoryStream.java @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import java.io.IOException; +import java.nio.file.DirectoryIteratorException; +import java.nio.file.DirectoryStream; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Iterator; +import java.util.NoSuchElementException; +import java.util.Set; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobListDetails; +import com.azure.storage.blob.models.ListBlobsOptions; + +/** + * A type for iterating over the contents of a directory. + * + * This type is asynchronously closeable, i.e. closing the stream from any thread will cause the stream to stop + * returning elements at that point. + * + * {@inheritDoc} + */ +public final class AzureDirectoryStream implements DirectoryStream { + private static final ClientLogger LOGGER = new ClientLogger(AzureDirectoryStream.class); + + private final AzurePath path; + private final DirectoryStream.Filter filter; + private boolean iteratorRequested = false; + private final AzureDirectoryIterator iterator; + boolean closed = false; + + AzureDirectoryStream(AzurePath path, DirectoryStream.Filter filter) throws IOException { + this.path = path; + this.filter = filter; + this.iterator = new AzureDirectoryIterator(this, this.path, this.filter); + } + + @Override + public Iterator iterator() { + if (this.iteratorRequested) { + throw LoggingUtility.logError(LOGGER, + new IllegalStateException("Only one iterator may be requested from a given directory stream")); + } + this.iteratorRequested = true; + return this.iterator; + } + + @Override + public void close() throws IOException { + this.closed = true; + } + + private static class AzureDirectoryIterator implements Iterator { + private static final ClientLogger LOGGER = new ClientLogger(AzureDirectoryIterator.class); + + private final AzureDirectoryStream parentStream; + private final DirectoryStream.Filter filter; + private final Iterator blobIterator; + private final AzurePath path; + private final Path withoutRoot; + private Path bufferedNext = null; + private final Set directoryPaths; + + AzureDirectoryIterator(AzureDirectoryStream parentStream, AzurePath path, + DirectoryStream.Filter filter) throws IOException { + this.parentStream = parentStream; + this.filter = filter; + this.path = path; + + /* + Resolving two paths requires that either both have a root or neither does. Because the paths returned from + listing will never have a root, we prepare a copy of the list path without a root for quick resolving later. + */ + Path root = this.path.getRoot(); + this.withoutRoot = root == null ? this.path : root.relativize(this.path); + + directoryPaths = new HashSet<>(); + + BlobContainerClient containerClient; + ListBlobsOptions listOptions = new ListBlobsOptions() + .setDetails(new BlobListDetails().setRetrieveMetadata(true)); + if (path.isRoot()) { + String containerName = path.toString().substring(0, path.toString().length() - 1); + AzureFileSystem afs = ((AzureFileSystem) path.getFileSystem()); + containerClient = ((AzureFileStore) afs.getFileStore()).getContainerClient(); + } else { + AzureResource azureResource = new AzureResource(path); + listOptions.setPrefix(azureResource.getBlobClient().getBlobName() + AzureFileSystem.PATH_SEPARATOR); + containerClient = azureResource.getContainerClient(); + } + this.blobIterator = containerClient + .listBlobsByHierarchy(AzureFileSystem.PATH_SEPARATOR, listOptions, null).iterator(); + } + + @Override + public boolean hasNext() { + AzurePath.ensureFileSystemOpen(path); + + // Closing the parent stream halts iteration. + if (parentStream.closed) { + return false; + } + + // In case a customer calls hasNext multiple times in a row. If we've buffered an element, we have a next. + if (this.bufferedNext != null) { + return true; + } + + /* + Search for a new element that passes the filter and buffer it when found. If no such element is found, + return false. + */ + while (this.blobIterator.hasNext()) { + BlobItem nextBlob = this.blobIterator.next(); + Path nextPath = getNextListResult(nextBlob); + try { + if (this.filter.accept(nextPath) && isNotDuplicate(nextPath, nextBlob)) { + this.bufferedNext = nextPath; + return true; + } + } catch (IOException e) { + throw LoggingUtility.logError(LOGGER, new DirectoryIteratorException(e)); + } + } + return false; + } + + @Override + public Path next() { + if (this.bufferedNext == null) { + if (!this.hasNext()) { // This will populate bufferedNext in the process. + throw LoggingUtility.logError(LOGGER, new NoSuchElementException()); + } + } + Path next = this.bufferedNext; // bufferedNext will have been populated by hasNext() + this.bufferedNext = null; + return next; + } + + @Override + public void remove() { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException()); + } + + private Path getNextListResult(BlobItem blobItem) { + /* + Listing results return the full blob path, and we don't want to duplicate the path we listed off of, so + we relativize to remove it. + */ + String blobName = blobItem.getName(); + Path relativeResult = this.withoutRoot.relativize( + this.path.getFileSystem().getPath(blobName)); + + // Resolve the cleaned list result against the original path for the final result. + return this.path.resolve(relativeResult); + } + + /* + If there is a concrete directory with children, a given path will be returned twice: once as the marker blob + and once as the prefix for its children. We don't want to return the item twice, and we have no guarantees on + result ordering, so we have to maintain a cache of directory paths we've seen in order to de-dup. + */ + private boolean isNotDuplicate(Path path, BlobItem blob) { + /* + If the blob is not a prefix and the blob does not contain the directory metadata marker, it is a normal blob + and therefore will not be duplicated. + */ + if (!(blob.isPrefix() != null && blob.isPrefix()) + && !(blob.getMetadata() != null && blob.getMetadata().containsKey(AzureResource.DIR_METADATA_MARKER))) { + return true; + } + + // If the set contains this path, it means we've seen it before and we shouldn't return it again. + if (this.directoryPaths.contains(path.toString())) { + return false; + } + + // We haven't seen this before. Track it and indicate it should be returned. + this.directoryPaths.add(path.toString()); + return true; + } + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileStore.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileStore.java new file mode 100644 index 00000000000..bbe361864bc --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileStore.java @@ -0,0 +1,194 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobContainerClient; + +import java.io.IOException; +import java.nio.file.FileStore; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.FileStoreAttributeView; +import java.util.Objects; + +/** + * An {@code AzureFileStore} is a {@link FileStore} backed by an Azure Blob Storage container. + */ +public final class AzureFileStore extends FileStore { + private static final ClientLogger LOGGER = new ClientLogger(AzureFileStore.class); + + private static final String AZURE_FILE_STORE_TYPE = "AzureBlobContainer"; + + private final AzureFileSystem parentFileSystem; + private final BlobContainerClient containerClient; + + + AzureFileStore(AzureFileSystem parentFileSystem, String containerName, Boolean skipConnectionCheck) + throws IOException { + // A FileStore should only ever be created by a FileSystem. + if (Objects.isNull(parentFileSystem)) { + throw LoggingUtility.logError(LOGGER, new IllegalStateException("AzureFileStore cannot be instantiated " + + "without a parent FileSystem")); + } + this.parentFileSystem = parentFileSystem; + this.containerClient = this.parentFileSystem.getBlobServiceClient().getBlobContainerClient(containerName); + + if (skipConnectionCheck == null || !skipConnectionCheck) { + try { + // This also serves as our connection check. + if (!this.containerClient.exists()) { + this.containerClient.create(); + } + } catch (Exception e) { + throw LoggingUtility.logError(LOGGER, new IOException("There was an error in establishing the existence of " + + "container: " + containerName, e)); + } + } + } + + /** + * Returns the name of the container that underlies this file store. + * + * @return the name of the container that underlies this file store. + */ + @Override + public String name() { + return this.containerClient.getBlobContainerName(); + } + + /** + * Returns the {@code String "AzureBlobContainer"} to indicate that the file store is backed by a remote blob + * container in Azure Storage. + * + * @return {@code "AzureBlobContainer"} + */ + @Override + public String type() { + return AZURE_FILE_STORE_TYPE; + } + + /** + * Always returns false. + *

+ * It may be the case that the authentication method provided to this file system only + * supports read operations and hence the file store is implicitly read only in this view, but that does not + * imply the underlying container/file store is inherently read only. Creating/specifying read only file stores + * is not currently supported. + * + * @return false. + */ + @Override + public boolean isReadOnly() { + return false; + } + + /** + * Returns the size, in bytes, of the file store. + *

+ * Containers do not limit the amount of data stored. This method will always return max long. + * + * @return the size of the file store. + * @throws IOException If an I/O error occurs. + */ + @Override + public long getTotalSpace() throws IOException { + return Long.MAX_VALUE; + } + + /** + * Returns the number of bytes available to this Java virtual machine on the file store. + *

+ * Containers do not limit the amount of data stored. This method will always return max long. + * + * @return the number of bytes available on the file store. + * @throws IOException If an I/O error occurs. + */ + @Override + public long getUsableSpace() throws IOException { + return Long.MAX_VALUE; + } + + /** + * Returns the number of unallocated bytes in the file store. + *

+ * Containers do not limit the amount of data stored. This method will always return max long. + * + * @return the number of unallocated bytes in the file store. + * @throws IOException If an I/O error occurs. + */ + @Override + public long getUnallocatedSpace() throws IOException { + return Long.MAX_VALUE; + } + + /** + * Tells whether this file store supports the file attributes identified by the given file attribute view. + *

+ * All file stores in this file system support the following views: + *

    + *
  • {@link java.nio.file.attribute.BasicFileAttributeView}
    • + *
  • {@link AzureBasicFileAttributeView}
    • + *
  • {@link AzureBlobFileAttributeView}
    • + *
+ * + * @param type the file attribute view type + * @return Whether the file attribute view is supported. + */ + @Override + public boolean supportsFileAttributeView(Class type) { + return AzureFileSystem.SUPPORTED_ATTRIBUTE_VIEWS.containsKey(type); + } + + /** + * Tells whether this file store supports the file attributes identified by the given file attribute view. + *

+ * All file stores in this file system support the following views: + *

    + *
  • {@link java.nio.file.attribute.BasicFileAttributeView}
    • + *
  • {@link AzureBasicFileAttributeView}
    • + *
  • {@link AzureBlobFileAttributeView}
    • + *
+ * + * @param name the name of the file attribute view + * @return whether the file attribute view is supported. + */ + @Override + public boolean supportsFileAttributeView(String name) { + return AzureFileSystem.SUPPORTED_ATTRIBUTE_VIEWS.containsValue(name); + } + + /** + * Returns a FileStoreAttributeView of the given type. + *

+ * This method always returns null as no {@link FileStoreAttributeView} is currently supported. + * + * @param aClass a class + * @return null + */ + @Override + public V getFileStoreAttributeView(Class aClass) { + return null; + } + + /** + * Unsupported. + *

+ * This method always throws an {@code UnsupportedOperationException} as no {@link FileStoreAttributeView} is + * currently supported. + * + * @param s a string + * @return The attribute value. + * @throws UnsupportedOperationException unsupported + * @throws IOException never + */ + @Override + public Object getAttribute(String s) throws IOException { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException("FileStoreAttributeViews aren't" + + " supported.")); + } + + BlobContainerClient getContainerClient() { + return this.containerClient; + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystem.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystem.java new file mode 100644 index 00000000000..8ca4361bd3e --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystem.java @@ -0,0 +1,534 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import java.io.IOException; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.PathMatcher; +import java.nio.file.WatchService; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.attribute.UserPrincipalLookupService; +import java.nio.file.spi.FileSystemProvider; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.regex.PatternSyntaxException; + +import com.azure.core.credential.AzureSasCredential; +import com.azure.core.http.HttpClient; +import com.azure.core.http.policy.HttpLogDetailLevel; +import com.azure.core.http.policy.HttpPipelinePolicy; +import com.azure.core.util.CoreUtils; +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobServiceClient; +import com.azure.storage.blob.BlobServiceClientBuilder; +import com.azure.storage.blob.implementation.util.BlobUserAgentModificationPolicy; +import com.azure.storage.common.StorageSharedKeyCredential; +import com.azure.storage.common.policy.RequestRetryOptions; +import com.azure.storage.common.policy.RetryPolicyType; + +/** + * Implement's Java's {@link FileSystem} interface for Azure Blob Storage. + *

+ * The following behavior is specific to this FileSystem: + *

+ * In the hierarchy of this file system, an {@code AzureFileSystem} corresponds to an Azure Blob Storage account. A + * file store is represented by a container in the storage account. Each container has one root directory. + *

+ * Closing the file system will not block on outstanding operations. Any operations in progress will be allowed to + * terminate naturally after the file system is closed, though no further operations may be started after the parent + * file system is closed. + *

+ * All instance of {@code AzureFileSystem} are opened for read-write access. + *

+ * For a more complete description of the uses for the constants described here, please see the instructions for opening + * and configuring a FileSystem in the docs of {@link FileSystemProvider}. + */ +public final class AzureFileSystem extends FileSystem { + private static final ClientLogger LOGGER = new ClientLogger(AzureFileSystem.class); + + // Configuration constants for blob clients. + /** + * Expected type: String + */ + public static final String AZURE_STORAGE_SHARED_KEY_CREDENTIAL = "AzureStorageSharedKeyCredential"; + + /** + * Expected type: String + */ + public static final String AZURE_STORAGE_SAS_TOKEN_CREDENTIAL = "AzureStorageSasTokenCredential"; + + /** + * Expected type: String + */ + public static final String AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL = "AzureStoragePublicAccessCredential"; + + /** + * Expected type: com.azure.core.http.policy.HttpLogLevelDetail + */ + public static final String AZURE_STORAGE_HTTP_LOG_DETAIL_LEVEL = "AzureStorageHttpLogDetailLevel"; + + /** + * Expected type: Integer + */ + public static final String AZURE_STORAGE_MAX_TRIES = "AzureStorageMaxTries"; + + /** + * Expected type: Integer + */ + public static final String AZURE_STORAGE_TRY_TIMEOUT = "AzureStorageTryTimeout"; + + /** + * Expected type: Long + */ + public static final String AZURE_STORAGE_RETRY_DELAY_IN_MS = "AzureStorageRetryDelayInMs"; + + /** + * Expected type: Long + */ + public static final String AZURE_STORAGE_MAX_RETRY_DELAY_IN_MS = "AzureStorageMaxRetryDelayInMs"; + + /** + * Expected type: com.azure.storage.common.policy.RetryPolicyType + */ + public static final String AZURE_STORAGE_RETRY_POLICY_TYPE = "AzureStorageRetryPolicyType"; + + /** + * Expected type: String + */ + public static final String AZURE_STORAGE_SECONDARY_HOST = "AzureStorageSecondaryHost"; + + /** + * Expected type: Long + */ + public static final String AZURE_STORAGE_UPLOAD_BLOCK_SIZE = "AzureStorageUploadBlockSize"; + + /** + * Expected type: Integer + */ + public static final String AZURE_STORAGE_MAX_CONCURRENCY_PER_REQUEST = "AzureStorageMaxConcurrencyPerRequest"; + + /** + * Expected type: Long + */ + public static final String AZURE_STORAGE_PUT_BLOB_THRESHOLD = "AzureStoragePutBlobThreshold"; + + /** + * Expected type: Integer + */ + public static final String AZURE_STORAGE_DOWNLOAD_RESUME_RETRIES = "AzureStorageDownloadResumeRetries"; + + static final String AZURE_STORAGE_HTTP_CLIENT = "AzureStorageHttpClient"; // undocumented; for test. + static final String AZURE_STORAGE_HTTP_POLICIES = "AzureStorageHttpPolicies"; // undocumented; for test. + + /** + * Expected type: String + */ + public static final String AZURE_STORAGE_FILE_STORES = "AzureStorageFileStores"; + + /** + * Expected type: Boolean + */ + public static final String AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK = "AzureStorageSkipInitialContainerCheck"; + + static final String PATH_SEPARATOR = "/"; + + private static final Map PROPERTIES = + CoreUtils.getProperties("azure-storage-blob-nio.properties"); + private static final String SDK_NAME = "name"; + private static final String SDK_VERSION = "version"; + private static final String CLIENT_NAME = PROPERTIES.getOrDefault(SDK_NAME, "UnknownName"); + private static final String CLIENT_VERSION = PROPERTIES.getOrDefault(SDK_VERSION, "UnknownVersion"); + + static final Map, String> SUPPORTED_ATTRIBUTE_VIEWS; + static { + Map, String> map = new HashMap<>(); + map.put(BasicFileAttributeView.class, "basic"); + map.put(AzureBasicFileAttributeView.class, "azureBasic"); + map.put(AzureBlobFileAttributeView.class, "azureBlob"); + SUPPORTED_ATTRIBUTE_VIEWS = Collections.unmodifiableMap(map); + } + + private final AzureFileSystemProvider parentFileSystemProvider; + private final BlobServiceClient blobServiceClient; + private final Long blockSize; + private final Long putBlobThreshold; + private final Integer maxConcurrencyPerRequest; + private final Integer downloadResumeRetries; + private FileStore defaultFileStore; + private boolean closed; + + private AzureSasCredential currentActiveSasCredential; + private Instant expiry; + + AzureFileSystem(AzureFileSystemProvider parentFileSystemProvider, String endpoint, Map config) + throws IOException { + // A FileSystem should only ever be instantiated by a provider. + if (Objects.isNull(parentFileSystemProvider)) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("AzureFileSystem cannot be instantiated" + + " without a parent FileSystemProvider")); + } + this.parentFileSystemProvider = parentFileSystemProvider; + + // Read configurations and build client. + try { + this.blobServiceClient = this.buildBlobServiceClient(endpoint, config); + this.blockSize = (Long) config.get(AZURE_STORAGE_UPLOAD_BLOCK_SIZE); + this.putBlobThreshold = (Long) config.get(AZURE_STORAGE_PUT_BLOB_THRESHOLD); + this.maxConcurrencyPerRequest = (Integer) config.get(AZURE_STORAGE_MAX_CONCURRENCY_PER_REQUEST); + this.downloadResumeRetries = (Integer) config.get(AZURE_STORAGE_DOWNLOAD_RESUME_RETRIES); + this.currentActiveSasCredential = (AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL); + + // Initialize and ensure access to FileStores. + this.defaultFileStore = this.initializeFileStore(config); + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("There was an error parsing the " + + "configurations map. Please ensure all fields are set to a legal value of the correct type.", e)); + } catch (IOException e) { + throw LoggingUtility.logError(LOGGER, + new IOException("Initializing FileStores failed. FileSystem could not be opened.", e)); + } + + this.closed = false; + } + + /** + * Returns the provider that created this file system. + * + * @return the provider that created this file system. + */ + @Override + public FileSystemProvider provider() { + return this.parentFileSystemProvider; + } + + /** + * Closes this file system. + *

+ * After a file system is closed then all subsequent access to the file system, either by methods defined by this + * class or on objects associated with this file system, throw ClosedFileSystemException. If the file system is + * already closed then invoking this method has no effect. + *

+ * Closing the file system will not block on outstanding operations. Any operations in progress will be allowed to + * terminate naturally after the file system is closed, though no further operations may be started after the + * parent file system is closed. + *

+ * Once closed, a file system with the same identifier as the one closed may be re-opened. + * + * @throws IOException If an I/O error occurs. + */ + @Override + public void close() throws IOException { + this.closed = true; + this.parentFileSystemProvider.closeFileSystem(this.getFileSystemUrl() + "/" + defaultFileStore.name()); + } + + /** + * Tells whether this file system is open. + * + * @return whether this file system is open. + */ + @Override + public boolean isOpen() { + return !this.closed; + } + + /** + * Tells whether this file system allows only read-only access to its file stores. + *

+ * Always returns false. It may be the case that the authentication method provided to this file system only + * supports read operations and hence the file system is implicitly read only in this view, but that does not + * imply the underlying account/file system is inherently read only. Creating/specifying read only file + * systems is not supported. + * + * @return false + */ + @Override + public boolean isReadOnly() { + return false; + } + + /** + * Returns the name separator, represented as a string. + *

+ * The separator used in this file system is {@code "/"}. + * + * @return "/" + */ + @Override + public String getSeparator() { + return AzureFileSystem.PATH_SEPARATOR; + } + + /** + * Returns an object to iterate over the paths of the root directories. + *

+ * The list of root directories corresponds to the list of available file stores and therefore containers specified + * upon initialization. A root directory always takes the form {@code ":"}. This list will + * respect the parameters provided during initialization. + *

+ * If a finite list of containers was provided on start up, this list will not change during the lifetime of this + * object. If containers are added to the account after initialization, they will be ignored. If a container is + * deleted or otherwise becomes unavailable, its root directory will still be returned but operations to it will + * fail. + * + * @return an object to iterate over the paths of the root directories + */ + @Override + public Iterable getRootDirectories() { + /* + Should we add different initialization options later: + If the file system was set to use all containers in the account, the account will be re-queried and the + list may grow or shrink if containers were added or deleted. + */ + return Arrays.asList(this.getPath(defaultFileStore.name() + AzurePath.ROOT_DIR_SUFFIX)); + } + + /** + * Returns an object to iterate over the underlying file stores + *

+ * This list will respect the parameters provided during initialization. + *

+ * If a finite list of containers was provided on start up, this list will not change during the lifetime of this + * object. If containers are added to the account after initialization, they will be ignored. If a container is + * deleted or otherwise becomes unavailable, its root directory will still be returned but operations to it will + * fail. + */ + @Override + public Iterable getFileStores() { + /* + Should we add different initialization options later: + If the file system was set to use all containers in the account, the account will be re-queried and the + list may grow or shrink if containers were added or deleted. + */ + return Arrays.asList(defaultFileStore); + } + + /** + * Returns the set of the names of the file attribute views supported by this FileSystem. + *

+ * This file system supports the following views: + *

    + *
  • {@link java.nio.file.attribute.BasicFileAttributeView}
    • + *
  • {@link AzureBasicFileAttributeView}
    • + *
  • {@link AzureBlobFileAttributeView}
    • + *
+ */ + @Override + public Set supportedFileAttributeViews() { + return new HashSet<>(SUPPORTED_ATTRIBUTE_VIEWS.values()); + } + + /** + * Converts a path string, or a sequence of more that when joined form a path string, to a Path. + *

+ * If more does not specify any elements then the value of the first parameter is the path string to convert. If + * more specifies one or more elements than each non-empty string, including first, is considered to be a sequence + * of name elements (see Path) and is joined to form a path string. The more will be joined using the name + * separator. + *

+ * Each name element will be {@code String}-joined to the other elements by this file system's first path separator. + * Naming conventions and allowed characters are as + * defined + * by the Azure Blob Storage service. The root component is interpreted as the container name and all name elements + * are interpreted as a part of the blob name. The character {@code ':'} is only allowed in the root component and + * must be the last character of the root component. + * + * @param first the path string or initial part of the path string + * @param more additional strings to be joined to form the path string + * @throws InvalidPathException if the path string cannot be converted. + */ + @Override + public Path getPath(String first, String... more) { + return new AzurePath(this, first, more); + } + + /** + * Unsupported. + * + * @param s the matcher + * @throws UnsupportedOperationException unsupported. + * @throws IllegalArgumentException never + * @throws PatternSyntaxException never + */ + @Override + public PathMatcher getPathMatcher(String s) throws IllegalArgumentException, PatternSyntaxException { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException()); + } + + /** + * Unsupported. + * + * @throws UnsupportedOperationException unsupported. + */ + @Override + public UserPrincipalLookupService getUserPrincipalLookupService() { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException()); + } + + /** + * Unsupported. + * + * @throws UnsupportedOperationException unsupported. + * @throws IOException Never thrown. + */ + @Override + public WatchService newWatchService() throws IOException { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException()); + } + + String getFileSystemUrl() { + return this.blobServiceClient.getAccountUrl(); + } + + BlobServiceClient getBlobServiceClient() { + return this.blobServiceClient; + } + + private BlobServiceClient buildBlobServiceClient(String endpoint, Map config) { + BlobServiceClientBuilder builder = new BlobServiceClientBuilder() + .endpoint(endpoint); + + // Set the credentials. + if (config.containsKey(AZURE_STORAGE_SHARED_KEY_CREDENTIAL)) { + builder.credential((StorageSharedKeyCredential) config.get(AZURE_STORAGE_SHARED_KEY_CREDENTIAL)); + } else if (config.containsKey(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL)) { + builder.credential((AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL)); + this.setExpiryFromSAS((AzureSasCredential) config.get(AZURE_STORAGE_SAS_TOKEN_CREDENTIAL)); + } else if (config.containsKey(AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL)) { + // The Blob Service Client Builder requires at least one kind of authentication to make requests + // For public files however, this is unnecessary. This key-value pair is to denote the case + // explicitly when we supply a placeholder SAS credential to bypass this requirement. + builder.credential((AzureSasCredential) config.get(AZURE_STORAGE_PUBLIC_ACCESS_CREDENTIAL)); + } else { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException(String.format("No credentials were " + + "provided. Please specify one of the following when constructing an AzureFileSystem: %s, %s.", + AZURE_STORAGE_SHARED_KEY_CREDENTIAL, AZURE_STORAGE_SAS_TOKEN_CREDENTIAL))); + } + + // Configure options and client. + builder.httpLogOptions(BlobServiceClientBuilder.getDefaultHttpLogOptions() + .setLogLevel((HttpLogDetailLevel) config.get(AZURE_STORAGE_HTTP_LOG_DETAIL_LEVEL))); + + RequestRetryOptions retryOptions = new RequestRetryOptions( + (RetryPolicyType) config.get(AZURE_STORAGE_RETRY_POLICY_TYPE), + (Integer) config.get(AZURE_STORAGE_MAX_TRIES), + (Integer) config.get(AZURE_STORAGE_TRY_TIMEOUT), + (Long) config.get(AZURE_STORAGE_RETRY_DELAY_IN_MS), + (Long) config.get(AZURE_STORAGE_MAX_RETRY_DELAY_IN_MS), + (String) config.get(AZURE_STORAGE_SECONDARY_HOST)); + builder.retryOptions(retryOptions); + + builder.httpClient((HttpClient) config.get(AZURE_STORAGE_HTTP_CLIENT)); + + // Add BlobUserAgentModificationPolicy + builder.addPolicy(new BlobUserAgentModificationPolicy(CLIENT_NAME, CLIENT_VERSION)); + + if (config.containsKey(AZURE_STORAGE_HTTP_POLICIES)) { + for (HttpPipelinePolicy policy : (HttpPipelinePolicy[]) config.get(AZURE_STORAGE_HTTP_POLICIES)) { + builder.addPolicy(policy); + } + } + + return builder.buildClient(); + } + + private FileStore initializeFileStore(Map config) throws IOException { + String fileStoreName = (String) config.get(AZURE_STORAGE_FILE_STORES); + if (CoreUtils.isNullOrEmpty(fileStoreName)) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("The list of FileStores cannot be " + + "null.")); + } + + Boolean skipConnectionCheck = (Boolean) config.get(AZURE_STORAGE_SKIP_INITIAL_CONTAINER_CHECK); + Map fileStores = new HashMap<>(); + this.defaultFileStore = new AzureFileStore(this, fileStoreName, skipConnectionCheck); + return this.defaultFileStore; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AzureFileSystem that = (AzureFileSystem) o; + return Objects.equals(this.getFileSystemUrl(), that.getFileSystemUrl()); + } + + @Override + public int hashCode() { + return Objects.hash(this.getFileSystemUrl()); + } + + Path getDefaultDirectory() { + return this.getPath(this.defaultFileStore.name() + AzurePath.ROOT_DIR_SUFFIX); + } + + FileStore getFileStore() throws IOException { + if (this.defaultFileStore == null) { + throw LoggingUtility.logError(LOGGER, new IOException("FileStore not initialized")); + } + return defaultFileStore; + } + + Long getBlockSize() { + return this.blockSize; + } + + Long getPutBlobThreshold() { + return this.putBlobThreshold; + } + + Integer getMaxConcurrencyPerRequest() { + return this.maxConcurrencyPerRequest; + } + + public String createSASAppendedURL(String url) throws IllegalStateException { + if (Objects.isNull(currentActiveSasCredential)) { + throw new IllegalStateException("No current active SAS credential present"); + } + return url + "?" + currentActiveSasCredential.getSignature(); + } + + public Optional getExpiry() { + return Optional.ofNullable(expiry); + } + + private void setExpiryFromSAS(AzureSasCredential token) { + List strings = Arrays.asList(token.getSignature().split("&")); + Optional expiryString = strings.stream() + .filter(s -> s.startsWith("se")) + .findFirst() + .map(s -> s.replaceFirst("se=","")) + .map(s -> s.replace("%3A", ":")); + this.expiry = expiryString.map(es -> Instant.parse(es)).orElse(null); + } + + /** + * Return true if this filesystem has SAS credentials with an expiration data attached, and we're within + * `buffer` of the expiration. Return false if our credentials don't come with an expiration, or we + * aren't within `buffer` of our expiration. + */ + public boolean isExpired(Duration buffer) { + return Optional.ofNullable(this.expiry) + .map(e -> Instant.now().plus(buffer).isAfter(e)) + .orElse(false); + + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystemProvider.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystemProvider.java new file mode 100644 index 00000000000..2066acf89d5 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureFileSystemProvider.java @@ -0,0 +1,1197 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.UncheckedIOException; +import java.net.HttpURLConnection; +import java.net.URI; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.AccessDeniedException; +import java.nio.file.AccessMode; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryNotEmptyException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileAlreadyExistsException; +import java.nio.file.FileStore; +import java.nio.file.FileSystem; +import java.nio.file.FileSystemAlreadyExistsException; +import java.nio.file.FileSystemNotFoundException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.NoSuchFileException; +import java.nio.file.NotDirectoryException; +import java.nio.file.OpenOption; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.nio.file.StandardOpenOption; +import java.nio.file.attribute.BasicFileAttributeView; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.FileAttributeView; +import java.nio.file.spi.FileSystemProvider; +import java.time.Duration; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.function.Consumer; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import com.azure.core.util.CoreUtils; +import com.azure.core.util.logging.ClientLogger; +import com.azure.core.util.polling.SyncPoller; +import com.azure.storage.blob.models.BlobCopyInfo; +import com.azure.storage.blob.models.BlobErrorCode; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.ParallelTransferOptions; + +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; + +/** + * The {@code AzureFileSystemProvider} is Azure Storage's implementation of the nio interface on top of Azure Blob + * Storage. + *

+ * Particular care should be taken when working with a remote storage service. This implementation makes no guarantees + * on behavior or state should other processes operate on the same data concurrently; file systems from this provider + * will assume they have exclusive access to their data and will behave without regard for potential of interfering + * applications. Moreover, remote file stores introduce higher latencies. Therefore, additional consideration should be + * given to managing concurrency: race conditions are more likely to manifest and network failures occur more frequently + * than disk failures. These and other such distributed application scenarios must be considered when working with this + * file system. While the {@code AzureFileSystem} will ensure it takes appropriate steps towards robustness and + * reliability, the application developer must design around these failure scenarios and have fallback and retry options + * available. + *

+ * The Azure Blob Storage service backing these APIs is not a true FileSystem, nor is it the goal of this implementation + * to force Azure Blob Storage to act like a full-fledged file system. Some APIs and scenarios will remain unsupported + * indefinitely until they may be sensibly implemented. Other APIs may experience lower performance than is expected + * because of the number of network requests needed to ensure correctness. The javadocs for each type and method should + * also be read carefully to understand what guarantees are made and how they may differ from the contract defined by + * {@link FileSystemProvider}. + *

+ * The scheme for this provider is {@code "azb"}, and the format of the URI to identify an {@code AzureFileSystem} is + * {@code "azb://?endpoint="}. The endpoint of the Storage account is used to uniquely identify the + * filesystem. + *

+ * An {@link AzureFileSystem} is backed by an account. An {@link AzureFileStore} is backed by a container. Any number of + * containers may be specified as file stores upon creation of the file system. When a file system is created, + * it will try to retrieve the properties of each container to ensure connection to the account. If any of the + * containers does not exist, it will be created. Failure to access or create containers as necessary will result in + * an exception and failure to create the file system. Any data existing in the containers will be preserved and + * accessible via the file system, though customers should be aware that it must be in a format understandable by + * the types in this package or behavior will be undefined. + *

+ * {@link #newFileSystem(URI, Map)} will check for the following keys in the configuration map and expect the named + * types. Any entries not listed here will be ignored. Note that {@link AzureFileSystem} has public constants defined + * for each of the keys for convenience. Most values are documented in the blob package. Any values which are unique to + * nio will be documented here. + *

    + *
  • {@code AzureStorageSharedKeyCredential:}{@link com.azure.storage.common.StorageSharedKeyCredential}
    • + *
  • {@code AzureStorageSasTokenCredential:}{@link com.azure.core.credential.AzureSasCredential}
    • + *
  • {@code AzureStorageHttpLogDetailLevel:}{@link com.azure.core.http.policy.HttpLogDetailLevel}
    • + *
  • {@code AzureStorageMaxTries:}{@link Integer}
    • + *
  • {@code AzureStorageTryTimeout:}{@link Integer}
    • + *
  • {@code AzureStorageRetryDelayInMs:}{@link Long}
    • + *
  • {@code AzureStorageMaxRetryDelayInMs:}{@link Long}
    • + *
  • {@code AzureStorageRetryPolicyType:}{@link com.azure.storage.common.policy.RetryPolicyType}
    • + *
  • {@code AzureStorageSecondaryHost:}{@link String}
    • + *
  • {@code AzureStorageSecondaryHost:}{@link Integer}
    • + *
  • {@code AzureStorageBlockSize:}{@link Long}
    • + *
  • {@code AzureStoragePutBlobThreshold:}{@link Long}
    • + *
  • {@code AzureStorageMaxConcurrencyPerRequest:}{@link Integer}
    • + *
  • {@code AzureStorageDownloadResumeRetries:}{@link Integer}
    • + *
  • {@code AzureStorageFileStores:}{@link String}
    • + *
  • {@code AzureStorageSkipInitialContainerCheck:}{@link Boolean}. Indicates that the initial check which + * confirms the existence of the containers meant to act as file stores should be skipped. This can be useful in + * cases where a sas token that is scoped to only one file is used to authenticate.
    • + *
+ *

+ * Either an account key or a sas token must be specified. If both are provided, the account key will be preferred. If + * a sas token is specified, the customer must take care that it has appropriate permissions to perform the actions + * demanded of the file system in a given workflow, including the initial connection check specified above. The same + * token will be applied to all operations. + *

+ * An iterable of file stores must also be provided; each entry should simply be the name of a container. The first + * container listed will be considered the default file store and the root directory of which will be the file system's + * default directory. All other values listed are used to configure the underlying + * {@link com.azure.storage.blob.BlobServiceClient}. Please refer to that type for more information on these values. + * + * @see FileSystemProvider + */ +public final class AzureFileSystemProvider extends FileSystemProvider { + /* + * A static inner class is used to hold the ClientLogger for AzureFileSystemProvider to defer creating the + * ClientLogger until logging is needed. Some implementations of SLF4J may make calls to load FileSystemProviders + * which results in a load FileSystemProviders to occur during a call to load FileSystemProviders. This results in + * the JVM to throw an exception that a circular call to load FileSystemProviders has occurred. + */ + private static final class ClientLoggerHolder { + private static final ClientLogger LOGGER = new ClientLogger(AzureFileSystemProvider.class); + } + + /** + * A helper for setting the HTTP properties when creating a directory. + */ + public static final String CONTENT_TYPE = "Content-Type"; + + /** + * A helper for setting the HTTP properties when creating a directory. + */ + public static final String CONTENT_DISPOSITION = "Content-Disposition"; + + /** + * A helper for setting the HTTP properties when creating a directory. + */ + public static final String CONTENT_LANGUAGE = "Content-Language"; + + /** + * A helper for setting the HTTP properties when creating a directory. + */ + public static final String CONTENT_ENCODING = "Content-Encoding"; + + /** + * A helper for setting the HTTP properties when creating a directory. + */ + public static final String CONTENT_MD5 = "Content-MD5"; + + /** + * A helper for setting the HTTP properties when creating a directory. + */ + public static final String CACHE_CONTROL = "Cache-Control"; + + private static final String ENDPOINT_QUERY_KEY = "endpoint"; + private static final int COPY_TIMEOUT_SECONDS = 30; + private static final Set OUTPUT_STREAM_DEFAULT_OPTIONS = + Collections.unmodifiableSet(new HashSet<>(Arrays.asList(StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + StandardOpenOption.TRUNCATE_EXISTING))); + private static final Set OUTPUT_STREAM_SUPPORTED_OPTIONS = + Collections.unmodifiableSet(new HashSet<>(Arrays.asList( + StandardOpenOption.CREATE_NEW, + StandardOpenOption.CREATE, + StandardOpenOption.WRITE, + // Though we don't actually truncate, the same result is achieved by overwriting the destination. + StandardOpenOption.TRUNCATE_EXISTING))); + + private final ConcurrentMap openFileSystems; + + + // Specs require a public zero argument constructor. + /** + * Creates an AzureFileSystemProvider. + */ + public AzureFileSystemProvider() { + this.openFileSystems = new ConcurrentHashMap<>(); + } + + /** + * Returns the URI scheme that identifies this provider: {@code "azb".} + * + * @return {@code "azb"} + */ + @Override + public String getScheme() { + return "azb"; + } + + /** + * Constructs a new FileSystem object identified by a URI. + *

+ * The format of a {@code URI} identifying a file system is {@code "azb://?endpoint="}. + *

+ * Once closed, a file system with the same identifier may be reopened. + * + * @param uri URI reference + * @param config A map of provider specific properties to configure the file system + * @return a new file system. + * @throws IllegalArgumentException If the pre-conditions for the uri parameter aren't met, or the env parameter + * does not contain properties required by the provider, or a property value is invalid. + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + * @throws FileSystemAlreadyExistsException If the file system has already been created. + */ + @Override + public FileSystem newFileSystem(URI uri, Map config) throws IOException { + String endpoint = extractAccountEndpoint(uri); + + if (this.openFileSystems.containsKey(endpoint)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new FileSystemAlreadyExistsException("Name: " + endpoint)); + } + + AzureFileSystem afs = new AzureFileSystem(this, endpoint, config); + this.openFileSystems.put(endpoint, afs); + + return afs; + } + + /** + * Returns an existing FileSystem created by this provider. + *

+ * The format of a {@code URI} identifying a file system is {@code "azb://?endpoint=<endpoint>"}. + *

+ * Trying to retrieve a closed file system will throw a {@link FileSystemNotFoundException}. Once closed, a + * file system with the same identifier may be reopened. + * + * @param uri URI reference + * @return the file system + * @throws IllegalArgumentException If the pre-conditions for the uri parameter aren't met + * @throws FileSystemNotFoundException If the file system already exists + * @throws SecurityException never + */ + @Override + public FileSystem getFileSystem(URI uri) { + String endpoint = extractAccountEndpoint(uri); + if (!this.openFileSystems.containsKey(endpoint)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new FileSystemNotFoundException("Name: " + endpoint)); + } + return this.openFileSystems.get(endpoint); + } + + /** + * Return a Path object by converting the given URI. The resulting Path is associated with a FileSystem that already + * exists. + * + * @param uri The URI to convert + * @return The path identified by the URI. + * @throws IllegalArgumentException If the URI scheme does not identify this provider or other preconditions on the + * uri parameter do not hold + * @throws FileSystemNotFoundException if the file system identified by the query does not exist + * @throws SecurityException never + * + * @see #getFileSystem(URI) for information on the URI format + */ + @Override + public Path getPath(URI uri) { + return getFileSystem(uri).getPath(uri.getPath()); + } + + /** + * Opens or creates a file, returning a seekable byte channel to access the file. + *

+ * This method is primarily offered to support some jdk convenience methods such as + * {@link Files#createFile(Path, FileAttribute[])} which requires opening a channel and closing it. A channel may + * only be opened in read mode OR write mode. It may not be opened in read/write mode. Seeking is supported for + * reads, but not for writes. Modifications to existing files is not permitted--only creating new files or + * overwriting existing files. + *

+ * This type is not threadsafe to prevent having to hold locks across network calls. + * + * @param path the path of the file to open + * @param set options specifying how the file should be opened + * @param fileAttributes an optional list of file attributes to set atomically when creating the directory + * @return a new seekable byte channel + * @throws UnsupportedOperationException Operation is not supported. + * @throws IllegalArgumentException if the set contains an invalid combination of options + * @throws FileAlreadyExistsException if a file of that name already exists and the CREATE_NEW option is specified + * (optional specific exception) + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public SeekableByteChannel newByteChannel(Path path, Set set, + FileAttribute... fileAttributes) throws IOException { + if (Objects.isNull(set)) { + set = Collections.emptySet(); + } + + if (set.contains(StandardOpenOption.WRITE)) { + return new AzureSeekableByteChannel( + (NioBlobOutputStream) this.newOutputStreamInternal(path, set, fileAttributes), path); + } else { + return new AzureSeekableByteChannel( + (NioBlobInputStream) this.newInputStream(path, set.toArray(new OpenOption[0])), path); + } + } + + /** + * Opens an {@link InputStream} to the given path. + *

+ * The stream will not attempt to read or buffer the entire file. However, when fetching data, it will always + * request the same size chunk of several MB to prevent network thrashing on small reads. Mark and reset are + * supported. + *

+ * Only {@link StandardOpenOption#READ} is supported. Any other option will throw. + * + * @param path the path to the file to open + * @param options options specifying how the file is opened + * @return a new input stream + * @throws IllegalArgumentException if an invalid combination of options is specified + * @throws UnsupportedOperationException if an unsupported option is specified + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public InputStream newInputStream(Path path, OpenOption... options) throws IOException { + // Validate options. Only read is supported. + if (options.length > 1 || (options.length > 0 && !options[0].equals(StandardOpenOption.READ))) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new UnsupportedOperationException("Only the read option is supported.")); + } + + AzureResource resource = new AzureResource(path); + AzurePath.ensureFileSystemOpen(resource.getPath()); + + // Ensure the path points to a file. + if (!resource.checkDirStatus().equals(DirectoryStatus.NOT_A_DIRECTORY)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IOException("Path either does not exist or points to a directory." + + "Path must point to a file. Path: " + path.toString())); + } + + // Note that methods on BlobInputStream are already synchronized. + return new NioBlobInputStream(resource.getBlobClient().openInputStream(), resource.getPath()); + } + + /** + * Opens an {@link OutputStream} to the given path. The resulting file will be stored as a block blob. + *

+ * The only supported options are {@link StandardOpenOption#CREATE}, {@link StandardOpenOption#CREATE_NEW}, + * {@link StandardOpenOption#WRITE}, {@link StandardOpenOption#TRUNCATE_EXISTING}. Any other options will throw an + * {@link UnsupportedOperationException}. {@code WRITE} and {@code TRUNCATE_EXISTING} must be specified or an + * {@link IllegalArgumentException} will be thrown. Hence, files cannot be updated, only overwritten completely. + *

+ * This stream will not attempt to buffer the entire file, however some buffering will be done for potential + * optimizations and to avoid network thrashing. Specifically, up to + * {@link AzureFileSystem#AZURE_STORAGE_PUT_BLOB_THRESHOLD} bytes will be buffered initially. If that threshold is + * exceeded, the data will be broken into chunks and sent in blocks, and writes will be buffered into sizes of + * {@link AzureFileSystem#AZURE_STORAGE_UPLOAD_BLOCK_SIZE}. The maximum number of buffers of this size to be + * allocated is defined by {@link AzureFileSystem#AZURE_STORAGE_MAX_CONCURRENCY_PER_REQUEST}, which also configures + * the level of parallelism with which we may write and thus may affect write speeds as well. + *

+ * The data is only committed when the steam is closed. Hence, data cannot be read from the destination until the + * stream is closed. When the close method returns, it is guaranteed that, barring any errors, the data is finalized + * and available for reading. + *

+ * Writing happens asynchronously. Bytes passed for writing are stored until either the threshold or block size are + * met at which time they are sent to the service. When the write method returns, there is no guarantee about which + * phase of this process the data is in other than it has been accepted and will be written. Again, closing will + * guarantee that the data is written and available. + *

+ * Flush is a no-op as regards data transfers, but it can be used to check the state of the stream for errors. + * This can be a useful tool because writing happens asynchronously, and therefore an error from a previous write + * may not otherwise be thrown unless the stream is flushed, closed, or written to again. + * + * @param path the path to the file to open or create + * @param options options specifying how the file is opened + * @return a new output stream + * @throws IllegalArgumentException if an invalid combination of options is specified + * @throws UnsupportedOperationException if an unsupported option is specified + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public OutputStream newOutputStream(Path path, OpenOption... options) throws IOException { + return newOutputStreamInternal(path, new HashSet<>(Arrays.asList(options))); + } + + OutputStream newOutputStreamInternal(Path path, Set optionsSet, + FileAttribute... fileAttributes) throws IOException { + // If options are empty, add Create, Write, TruncateExisting as defaults per nio docs. + if (optionsSet == null || optionsSet.size() == 0) { + optionsSet = OUTPUT_STREAM_DEFAULT_OPTIONS; + } + + // Check for unsupported options. + for (OpenOption option : optionsSet) { + if (!OUTPUT_STREAM_SUPPORTED_OPTIONS.contains(option)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new UnsupportedOperationException("Unsupported option: " + option.toString())); + } + } + + /* + Write must be specified. Either create_new or truncate must be specified. This is to ensure that no edits or + appends are allowed. + */ + if (!optionsSet.contains(StandardOpenOption.WRITE) + || !(optionsSet.contains(StandardOpenOption.TRUNCATE_EXISTING) + || optionsSet.contains(StandardOpenOption.CREATE_NEW))) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Write and either CreateNew or TruncateExisting must be specified to open " + + "an OutputStream")); + } + + AzureResource resource = new AzureResource(path); + AzurePath.ensureFileSystemOpen(resource.getPath()); + DirectoryStatus status = resource.checkDirStatus(); + + // Cannot write to a directory. + if (DirectoryStatus.isDirectory(status)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IOException("Cannot open an OutputStream to a directory. Path: " + path.toString())); + } + + // Writing to an empty location requires a create option. + if (status.equals(DirectoryStatus.DOES_NOT_EXIST) + && !(optionsSet.contains(StandardOpenOption.CREATE) + || optionsSet.contains(StandardOpenOption.CREATE_NEW))) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IOException("Writing to an empty location requires a create option. Path: " + path.toString())); + } + + // Cannot write to an existing file if create new was specified. + if (status.equals(DirectoryStatus.NOT_A_DIRECTORY) && optionsSet.contains(StandardOpenOption.CREATE_NEW)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IOException("A file already exists at this location and " + + "CREATE_NEW was specified. Path: " + path.toString())); + } + + // Create options based on file system config + AzureFileSystem fs = (AzureFileSystem) (path.getFileSystem()); + Integer blockSize = fs.getBlockSize() == null ? null : fs.getBlockSize().intValue(); + Integer putBlobThreshold = fs.getPutBlobThreshold() == null ? null : fs.getPutBlobThreshold().intValue(); + ParallelTransferOptions pto = new ParallelTransferOptions(blockSize, fs.getMaxConcurrencyPerRequest(), null, + putBlobThreshold); + + // Add an extra etag check for create new + BlobRequestConditions rq = null; + if (optionsSet.contains(StandardOpenOption.CREATE_NEW)) { + rq = new BlobRequestConditions().setIfNoneMatch("*"); + } + + // For parsing properties and metadata + if (fileAttributes == null) { + fileAttributes = new FileAttribute[0]; + } + resource.setFileAttributes(Arrays.asList(fileAttributes)); + + return new NioBlobOutputStream(resource.getBlobOutputStream(pto, rq), resource.getPath()); + } + + /** + * Returns an {@link AzureDirectoryStream} for iterating over the contents of a directory. The elements returned by + * the directory stream's iterator are of type Path, each one representing an entry in the directory. The Path + * objects are obtained as if by resolving the name of the directory entry against dir. The entries returned by the + * iterator are filtered by the given filter. + *

+ * When not using the try-with-resources construct, then directory stream's close method should be invoked after + * iteration is completed to free any resources held for the open directory. + *

+ * Where the filter terminates due to an uncaught error or runtime exception then it is propagated to the hasNext or + * next method. Where an IOException is thrown, it results in the hasNext or next method throwing a + * DirectoryIteratorException with the IOException as the cause. + * + * @param path the path to the directory + * @param filter the directory stream filter + * @return a new and open {@code DirectoryStream} object + * @throws IllegalArgumentException If the path type is not an instance of {@link AzurePath}. + * @throws NotDirectoryException if the file could not otherwise be opened because it is not a directory + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public DirectoryStream newDirectoryStream(Path path, DirectoryStream.Filter filter) + throws IOException { + if (!(path instanceof AzurePath)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("This provider cannot operate on subtypes of Path other than AzurePath")); + } + AzurePath.ensureFileSystemOpen(path); + + /* + Ensure the path is a directory. Note that roots are always directories. The case of an invalid root will be + caught in instantiating the stream below. + + Possible optimization later is to save the result of the list call to use as the first list call inside the + stream rather than a list call for checking the status and a list call for listing. + */ + if (!((AzurePath) path).isRoot() && !(new AzureResource(path).checkDirectoryExists())) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new NotDirectoryException(path.toString())); + } + + return new AzureDirectoryStream((AzurePath) path, filter); + } + + /** + * Creates a new directory at the specified path. + *

+ * The existence of a directory in the {@code AzureFileSystem} is defined on two levels. Weak existence is + * defined by the presence of a non-zero number of blobs prefixed with the directory's path. This concept is also + * known as a virtual directory and enables the file system to work with containers that were pre-loaded + * with data by another source but need to be accessed by this file system. Strong existence is defined as + * the presence of an actual storage resource at the given path, which in the case of directories, is a zero-length + * blob whose name is the directory path with a particular metadata field indicating the blob's status as a + * directory. This is also known as a concrete directory. Directories created by this file system will + * strongly exist. Operations targeting directories themselves as the object (e.g. setting properties) will target + * marker blobs underlying concrete directories. Other operations (e.g. listing) will operate on the blob-name + * prefix. + *

+ * This method fulfills the nio contract of: "The check for the existence of the file and the creation of the + * directory if it does not exist are a single operation that is atomic with respect to all other filesystem + * activities that might affect the directory." More specifically, this method will atomically check for strong + * existence of another file or directory at the given path and fail if one is present. On the other hand, we + * only check for weak existence of the parent to determine if the given path is valid. Additionally, the + * action of checking whether the parent exists, is not atomic with the creation of the directory. Note that + * while it is possible that the parent may be deleted between when the parent is determined to exist and the + * creation of the child, the creation of the child will always ensure the existence of a virtual parent, so the + * child will never be left floating and unreachable. The different checks on parent and child is due to limitations + * in the Storage service API. + *

+ * There may be some unintuitive behavior when working with directories in this file system, particularly virtual + * directories (usually those not created by this file system). A virtual directory will disappear as soon as all + * its children have been deleted. Furthermore, if a directory with the given path weakly exists at the time of + * calling this method, this method will still return success and create a concrete directory at the target + * location. In other words, it is possible to "double create" a directory if it first weakly exists and then is + * strongly created. This is both because it is impossible to atomically check if a virtual directory exists while + * creating a concrete directory and because such behavior will have minimal side effects--no files will be + * overwritten and the directory will still be available for writing as intended, though it may not be empty. This + * is not a complete list of such unintuitive behavior. + *

+ * This method will attempt to extract standard HTTP content headers from the list of file attributes to set them + * as blob headers. All other attributes will be set as blob metadata. The value of every attribute will be + * converted to a {@code String} except the Content-MD5 attribute which expects a {@code byte[]}. + * When extracting the content headers, the following strings will be used for comparison (constants for these + * values can be found on this type): + *

    + *
  • {@code Content-Type}
    • + *
  • {@code Content-Disposition}
    • + *
  • {@code Content-Language}
    • + *
  • {@code Content-Encoding}
    • + *
  • {@code Content-MD5}
    • + *
  • {@code Cache-Control}
    • + *
+ * Note that these properties also have a particular semantic in that if one is specified, all are updated. In other + * words, if any of the above is set, all those that are not set will be cleared. See the + * Azure Docs for more + * information. + * + * @param path the directory to create + * @param fileAttributes an optional list of file attributes to set atomically when creating the directory + * @throws IllegalArgumentException If the path type is not an instance of {@link AzurePath}. + * @throws UnsupportedOperationException if the array contains an attribute that cannot be set atomically when + * creating the directory + * @throws FileAlreadyExistsException if a directory could not otherwise be created because a file of that name + * already exists + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public void createDirectory(Path path, FileAttribute... fileAttributes) throws IOException { + fileAttributes = fileAttributes == null ? new FileAttribute[0] : fileAttributes; + + // Get the destination for the directory. Will throw if path is a root. + AzureResource azureResource = new AzureResource(path); + AzurePath.ensureFileSystemOpen(azureResource.getPath()); + + // Check if parent exists. If it does, atomically check if a file already exists and create a new dir if not. + if (azureResource.checkParentDirectoryExists()) { + try { + azureResource.setFileAttributes(Arrays.asList(fileAttributes)) + .putDirectoryBlob(new BlobRequestConditions().setIfNoneMatch("*")); + } catch (BlobStorageException e) { + if (e.getStatusCode() == HttpURLConnection.HTTP_CONFLICT + && e.getErrorCode().equals(BlobErrorCode.BLOB_ALREADY_EXISTS)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new FileAlreadyExistsException(azureResource.getPath().toString())); + } else { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IOException("An error occurred when creating the directory", e)); + } + } + } else { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IOException("Parent directory does not exist for path: " + azureResource.getPath())); + } + } + + /** + * Deletes the specified resource. + *

+ * This method is not atomic with respect to other file system operations. It is possible to delete a file in use by + * another process, and doing so will not immediately invalidate any channels open to that file--they will simply + * start to fail. Root directories cannot be deleted even when empty. + * + * @param path the path to the file to delete + * @throws IllegalArgumentException If the path type is not an instance of {@link AzurePath}. + * @throws NoSuchFileException if the file does not exist + * @throws DirectoryNotEmptyException if the file is a directory and could not otherwise be deleted because the + * directory is not empty + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public void delete(Path path) throws IOException { + // Basic validation. Must be an AzurePath. Cannot be a root. + AzureResource azureResource = new AzureResource(path); + AzurePath.ensureFileSystemOpen(azureResource.getPath()); + + // Check directory status--possibly throw DirectoryNotEmpty or NoSuchFile. + DirectoryStatus dirStatus = azureResource.checkDirStatus(); + if (dirStatus.equals(DirectoryStatus.DOES_NOT_EXIST)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new NoSuchFileException(path.toString())); + } + if (dirStatus.equals(DirectoryStatus.NOT_EMPTY)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new DirectoryNotEmptyException(path.toString())); + } + + // After all validation has completed, delete the resource. + try { + azureResource.getBlobClient().delete(); + } catch (BlobStorageException e) { + if (e.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new NoSuchFileException(path.toString())); + } + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new IOException(e)); + } + } + + /** + * Copies the resource at the source location to the destination. + *

+ * This method is not atomic with respect to other file system operations. More specifically, the checks necessary + * to validate the inputs and state of the file system are not atomic with the actual copying of data. If the copy + * is triggered, the copy itself is atomic and only a complete copy will ever be left at the destination. + *

+ * In addition to those in the docs for {@link FileSystemProvider#copy(Path, Path, CopyOption...)}, this method has + * the following requirements for successful completion. {@link StandardCopyOption#COPY_ATTRIBUTES} must be passed + * as it is impossible not to copy blob properties; if this option is not passed, an + * {@link UnsupportedOperationException} will be thrown. Neither the source nor the destination can be a root + * directory; if either is a root directory, an {@link IllegalArgumentException} will be thrown. The parent + * directory of the destination must at least weakly exist; if it does not, an {@link IOException} will be thrown. + * The only supported option other than {@link StandardCopyOption#COPY_ATTRIBUTES} is + * {@link StandardCopyOption#REPLACE_EXISTING}; the presence of any other option will result in an + * {@link UnsupportedOperationException}. + *

+ * This method supports both virtual and concrete directories as both the source and destination. Unlike when + * creating a directory, the existence of a virtual directory at the destination will cause this operation to fail. + * This is in order to prevent the possibility of overwriting a non-empty virtual directory with a file. Still, as + * mentioned above, this check is not atomic with the creation of the resultant directory. + * + * @param source the path to the file to copy + * @param destination the path to the target file + * @param copyOptions specifying how the copy should be done + * @throws UnsupportedOperationException if the array contains a copy option that is not supported + * @throws FileAlreadyExistsException if the target file exists but cannot be replaced because the REPLACE_EXISTING + * option is not specified + * @throws DirectoryNotEmptyException the REPLACE_EXISTING option is specified but the file cannot be replaced + * because it is a non-empty directory + * @throws IOException If an I/O error occurs. + * @throws IllegalArgumentException If the path type is not an instance of {@link AzurePath}. + * @throws SecurityException never + * @see #createDirectory(Path, FileAttribute[]) for more information about directory existence. + */ + @Override + public void copy(Path source, Path destination, CopyOption... copyOptions) throws IOException { + // If paths point to the same file, operation is a no-op. + if (source.equals(destination)) { + return; + } + + // Read and validate options. + // Remove accepted options as we find them. Anything left we don't support. + boolean replaceExisting = false; + List optionsList = new ArrayList<>(Arrays.asList(copyOptions)); +// NOTE: We're going to assume COPY_ATTRIBUTES as a default copy option (but can still be provided and handled safely) +// REPLACE_EXISTING must still be provided if you want to replace existing file + +// if (!optionsList.contains(StandardCopyOption.COPY_ATTRIBUTES)) { +// throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new UnsupportedOperationException( +// "StandardCopyOption.COPY_ATTRIBUTES must be specified as the service will always copy " +// + "file attributes.")); +// } + if(optionsList.contains(StandardCopyOption.COPY_ATTRIBUTES)) { + optionsList.remove(StandardCopyOption.COPY_ATTRIBUTES); + } + + if (optionsList.contains(StandardCopyOption.REPLACE_EXISTING)) { + replaceExisting = true; + optionsList.remove(StandardCopyOption.REPLACE_EXISTING); + } + + if (!optionsList.isEmpty()) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new UnsupportedOperationException("Unsupported copy option found. Only " + + "StandardCopyOption.COPY_ATTRIBUTES and StandardCopyOption.REPLACE_EXISTING are supported.")); + } + + // Validate paths. Build resources. + // Copying a root directory or attempting to create/overwrite a root directory is illegal. + AzureResource sourceRes = new AzureResource(source); + AzurePath.ensureFileSystemOpen(sourceRes.getPath()); + AzureResource destinationRes = new AzureResource(destination); + AzurePath.ensureFileSystemOpen(destinationRes.getPath()); + + // Check destination is not a directory with children. + DirectoryStatus destinationStatus = destinationRes.checkDirStatus(); + if (destinationStatus.equals(DirectoryStatus.NOT_EMPTY)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new DirectoryNotEmptyException(destination.toString())); + } + + /* + Set request conditions if we should not overwrite. We can error out here if we know something already exists, + but we will also create request conditions as a safeguard against overwriting something that was created + between our check and put. + */ + BlobRequestConditions requestConditions = null; + if (!replaceExisting) { + if (!destinationStatus.equals(DirectoryStatus.DOES_NOT_EXIST)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new FileAlreadyExistsException(destinationRes.getPath().toString())); + } + requestConditions = new BlobRequestConditions().setIfNoneMatch("*"); + } + + /* + More path validation + + Check that the parent for the destination exists. We only need to perform this check if there is nothing + currently at the destination, for if the destination exists, its parent at least weakly exists and we + can skip a service call. + */ + if (destinationStatus.equals(DirectoryStatus.DOES_NOT_EXIST) && !destinationRes.checkParentDirectoryExists()) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IOException("Parent directory of destination location does not exist. The destination path is " + + "therefore invalid. Destination: " + destinationRes.getPath())); + } + + /* + Try to copy the resource at the source path. + + There is an optimization here where we try to do the copy first and only check for a virtual directory if + there's a 404. In the cases of files and concrete directories, this only requires one request. For virtual + directories, however, this requires three requests: failed copy, check status, create directory. Depending on + customer scenarios and how many virtual directories they copy, it could be better to check the directory status + first and then do a copy or createDir, which would always be two requests for all resource types. + */ + + try { + /* + Format the url by appending the SAS token as a param, otherwise the copy request will fail. + AzureFileSystem has been updated to handle url transformation via createSASAuthorizedURL() + */ + AzureFileSystem afs = (AzureFileSystem) sourceRes.getPath().getFileSystem(); + String sasAppendedSourceUrl = afs.createSASAppendedURL(sourceRes.getBlobClient().getBlobUrl()); + SyncPoller pollResponse = + destinationRes.getBlobClient().beginCopy(sasAppendedSourceUrl, null, null, null, + null, requestConditions, null); + pollResponse.waitForCompletion(Duration.ofSeconds(COPY_TIMEOUT_SECONDS)); + } catch (BlobStorageException e) { + // If the source was not found, it could be because it's a virtual directory. Check the status. + // If a non-dir resource existed, it would have been copied above. This check is therefore sufficient. + if (e.getErrorCode().equals(BlobErrorCode.BLOB_NOT_FOUND) + && !sourceRes.checkDirStatus().equals(DirectoryStatus.DOES_NOT_EXIST)) { + /* + We already checked that the parent exists and validated the paths above, so we can put the blob + directly. + */ + destinationRes.putDirectoryBlob(requestConditions); + } else { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new IOException(e)); + } + } catch (RuntimeException e) { // To better log possible timeout from poller. + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new IOException(e)); + } + } + + // Used for checking the status of the root directory. To be implemented later when needed. + /*int checkRootDirStatus(BlobContainerClient rootClient) { + + }*/ + + /** + * Unsupported. + * + * @param path path + * @param path1 path + * @param copyOptions options + * @throws UnsupportedOperationException Operation is not supported. + */ + @Override + public void move(Path path, Path path1, CopyOption... copyOptions) throws IOException { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new UnsupportedOperationException()); + } + + /** + * Unsupported. + * + * @param path path + * @param path1 path + * @throws UnsupportedOperationException Operation is not supported. + */ + @Override + public boolean isSameFile(Path path, Path path1) throws IOException { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new UnsupportedOperationException()); + } + + /** + * Always returns false as hidden files are not supported. + * + * @param path the path + * @return false + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public boolean isHidden(Path path) throws IOException { + return false; + } + + /** + * Unsupported. + * + * @param path path + * @return the file store where the file is stored. + * @throws UnsupportedOperationException Operation is not supported. + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public FileStore getFileStore(Path path) throws IOException { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new UnsupportedOperationException()); + } + + /** + * Checks the existence, and optionally the accessibility, of a file. + *

+ * This method may only be used to check the existence of a file. It is not possible to determine the permissions + * granted to a given client, so if any mode argument is specified, an {@link UnsupportedOperationException} will be + * thrown. + * + * @param path the path to the file to check + * @param accessModes The access modes to check; may have zero elements + * @throws NoSuchFileException if a file does not exist + * @throws java.nio.file.AccessDeniedException the requested access would be denied or the access cannot be + * determined because the Java virtual machine has insufficient privileges or other reasons + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public void checkAccess(Path path, AccessMode... accessModes) throws IOException { + if (accessModes != null && accessModes.length != 0) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new AccessDeniedException("The access cannot be determined.")); + } + AzurePath.ensureFileSystemOpen(path); + + /* + Some static utility methods in the jdk require checking access on a root. ReadAttributes is not supported on + roots as they are containers. Furthermore, we always assume that roots exist as they are verified at creation + and cannot be deleted by the file system. Thus, we prefer a short circuit for roots. + */ + if (path instanceof AzurePath && ((AzurePath) path).isRoot()) { + return; + } + + // Read attributes already wraps BlobStorageException in an IOException. + try { + readAttributes(path, BasicFileAttributes.class); + } catch (IOException e) { + Throwable cause = e.getCause(); + if (cause instanceof BlobStorageException + && BlobErrorCode.BLOB_NOT_FOUND.equals(((BlobStorageException) cause).getErrorCode())) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new NoSuchFileException(path.toString())); + } else { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, e); + } + } + } + + /** + * Returns a file attribute view of a given type. + *

+ * See {@link AzureBasicFileAttributeView} and {@link AzureBlobFileAttributeView} for more information. + *

+ * Reading attributes on a virtual directory will return {@code null} for most properties other than + * {@link AzureBlobFileAttributes#isVirtualDirectory()}, which will return true. See + * {@link #createDirectory(Path, FileAttribute[])} for more information on virtual directories. + * + * @param path the path to the file + * @param type the Class object corresponding to the file attribute view + * @param linkOptions ignored + * @return a file attribute view of the specified type, or null if the attribute view type is not available + */ + @Override + @SuppressWarnings("unchecked") + public V getFileAttributeView(Path path, Class type, LinkOption... linkOptions) { + /* + No resource validation is necessary here. That can happen at the time of making a network requests internal to + the view object. + */ + if (type == BasicFileAttributeView.class || type == AzureBasicFileAttributeView.class) { + return (V) new AzureBasicFileAttributeView(path); + } else if (type == AzureBlobFileAttributeView.class) { + return (V) new AzureBlobFileAttributeView(path); + } else { + return null; + } + } + + /** + * Reads a file's attributes as a bulk operation. + *

+ * See {@link AzureBasicFileAttributes} and {@link AzureBlobFileAttributes} for more information. + *

+ * Reading attributes on a virtual directory will return {@code null} for most properties other than + * {@link AzureBlobFileAttributes#isVirtualDirectory()}, which will return true. See + * {@link #createDirectory(Path, FileAttribute[])} for more information on virtual directories. + * + * @param path the path to the file + * @param type the Class of the file attributes required to read + * @param linkOptions ignored + * @return the file attributes + * @throws UnsupportedOperationException if an attributes of the given type are not supported + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + @SuppressWarnings("unchecked") + public A readAttributes(Path path, Class type, LinkOption... linkOptions) + throws IOException { + AzurePath.ensureFileSystemOpen(path); + + Class view; + if (type == BasicFileAttributes.class || type == AzureBasicFileAttributes.class) { + view = AzureBasicFileAttributeView.class; + } else if (type == AzureBlobFileAttributes.class) { + view = AzureBlobFileAttributeView.class; + } else { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new UnsupportedOperationException()); + } + + /* + Resource validation will happen in readAttributes of the view. We don't want to double-check, and checking + internal to the view ensures it is always checked no matter which code path is taken. + */ + return (A) getFileAttributeView(path, view, linkOptions).readAttributes(); + } + + /** + * Reads a set of file attributes as a bulk operation. + *

+ * See {@link AzureBasicFileAttributes} and {@link AzureBlobFileAttributes} for more information. + *

+ * Reading attributes on a virtual directory will return {@code null} for all properties other than + * {@link AzureBlobFileAttributes#isVirtualDirectory()}, which will return true. See + * {@link #createDirectory(Path, FileAttribute[])} for more information on virtual directories. + * + * @param path the path to the file + * @param attributes the attributes to read + * @param linkOptions ignored + * @return a map of the attributes returned; may be empty. The map's keys are the attribute names, its values are + * the attribute values + * @throws UnsupportedOperationException if an attributes of the given type are not supported + * @throws IllegalArgumentException if no attributes are specified or an unrecognized attributes is specified + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public Map readAttributes(Path path, String attributes, LinkOption... linkOptions) + throws IOException { + if (attributes == null) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Attribute string cannot be null.")); + } + + AzurePath.ensureFileSystemOpen(path); + + Map results = new HashMap<>(); + + /* + AzureBlobFileAttributes can do everything the basic attributes can do and more. There's no need to instantiate + one of each if both are specified somewhere in the list as that will waste a network call. This can be + generified later if we need to add more attribute types, but for now we can stick to just caching the supplier + for a single attributes object. + */ + Map> attributeSuppliers = null; // Initialized later as needed. + String viewType; + String attributeList; + String[] parts = attributes.split(":"); + + if (parts.length > 2) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Invalid format for attribute string: " + attributes)); + } + + if (parts.length == 1) { + viewType = "basic"; // Per jdk docs. + attributeList = attributes; + } else { + viewType = parts[0]; + attributeList = parts[1]; + } + + /* + For specificity, our basic implementation of BasicFileAttributes uses the name azureBasic. However, the docs + state that "basic" must be supported, so we funnel to azureBasic. + */ + if ("basic".equals(viewType)) { + viewType = AzureBasicFileAttributeView.NAME; + } + if (!viewType.equals(AzureBasicFileAttributeView.NAME) && !viewType.equals(AzureBlobFileAttributeView.NAME)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new UnsupportedOperationException("Invalid attribute view: " + viewType)); + } + + for (String attributeName : attributeList.split(",")) { + /* + We rely on the azureBlobFAV to actually do the work here as mentioned above, but if basic is specified, we + should at least validate that the attribute is available on a basic view. + */ + // TODO: Put these strings in constants + if (viewType.equals(AzureBasicFileAttributeView.NAME)) { + if (!AzureBasicFileAttributes.ATTRIBUTE_STRINGS.contains(attributeName) && !"*".equals(attributeName)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Invalid attribute. View: " + viewType + + ". Attribute: " + attributeName)); + } + } + + // As mentioned, azure blob can fulfill requests to both kinds of views. + // Populate the supplier if we haven't already. + if (attributeSuppliers == null) { + attributeSuppliers = AzureBlobFileAttributes.getAttributeSuppliers( + this.readAttributes(path, AzureBlobFileAttributes.class, linkOptions)); + } + + // If "*" is specified, add all the attributes from the specified set. + if ("*".equals(attributeName)) { + if (viewType.equals(AzureBasicFileAttributeView.NAME)) { + for (String attr : AzureBasicFileAttributes.ATTRIBUTE_STRINGS) { + results.put(attr, attributeSuppliers.get(attr).get()); + } + } else { + // attributeSuppliers is guaranteed to have been set by this point. + for (Map.Entry> entry: attributeSuppliers.entrySet()) { + results.put(entry.getKey(), entry.getValue().get()); + } + } + + } else if (!attributeSuppliers.containsKey(attributeName)) { + // Validate that the attribute is legal and add the value returned by the supplier to the results. + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Invalid attribute. View: " + viewType + + ". Attribute: " + attributeName)); + } else { + results.put(attributeName, attributeSuppliers.get(attributeName).get()); + + } + } + + // Throw if nothing specified per jdk docs. + if (results.isEmpty()) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("No attributes were specified. Attributes: " + attributes)); + } + + return results; + } + + /** + * Sets the value of a file attribute. + *

+ * See {@link AzureBlobFileAttributeView} for more information. + *

+ * Setting attributes on a virtual directory is not supported and will throw an {@link IOException}. See + * {@link #createDirectory(Path, FileAttribute[])} for more information on virtual directories. + * + * @param path the path to the file + * @param attributes the attribute to set + * @param value the attribute value + * @param linkOptions ignored + * @throws UnsupportedOperationException if an attribute view is not available + * @throws IllegalArgumentException if the attribute name is not specified, or is not recognized, or the attribute + * value is of the correct type but has an inappropriate value + * @throws ClassCastException If the attribute value is not of the expected type or is a collection containing + * elements that are not of the expected type + * @throws IOException If an I/O error occurs. + * @throws SecurityException never + */ + @Override + public void setAttribute(Path path, String attributes, Object value, LinkOption... linkOptions) throws IOException { + AzurePath.ensureFileSystemOpen(path); + String viewType; + String attributeName; + String[] parts = attributes.split(":"); + if (parts.length > 2) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Invalid format for attribute string: " + attributes)); + } + if (parts.length == 1) { + viewType = "basic"; // Per jdk docs. + attributeName = attributes; + } else { + viewType = parts[0]; + attributeName = parts[1]; + } + + /* + For specificity, our basic implementation of BasicFileAttributes uses the name azureBasic. However, the docs + state that "basic" must be supported, so we funnel to azureBasic. + */ + if ("basic".equals(viewType)) { + viewType = AzureBasicFileAttributeView.NAME; + } + + // We don't actually support any setters on the basic view. + if (viewType.equals(AzureBasicFileAttributeView.NAME)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Invalid attribute. View: " + viewType + + ". Attribute: " + attributeName)); + } else if (viewType.equals(AzureBlobFileAttributeView.NAME)) { + Map> attributeConsumers = AzureBlobFileAttributeView.setAttributeConsumers( + this.getFileAttributeView(path, AzureBlobFileAttributeView.class, linkOptions)); + if (!attributeConsumers.containsKey(attributeName)) { + // Validate that the attribute is legal and add the value returned by the supplier to the results. + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("Invalid attribute. View: " + viewType + + ". Attribute: " + attributeName)); + } + try { + attributeConsumers.get(attributeName).accept(value); + } catch (UncheckedIOException e) { + if (e.getMessage().equals(AzureBlobFileAttributeView.ATTR_CONSUMER_ERROR)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, e.getCause()); + } + } + } else { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new UnsupportedOperationException("Invalid attribute view: " + viewType)); + } + } + + void closeFileSystem(String fileSystemName) { + this.openFileSystems.remove(fileSystemName); + } + + private String extractAccountEndpoint(URI uri) { + if (!uri.getScheme().equals(this.getScheme())) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, new IllegalArgumentException( + "URI scheme does not match this provider")); + } + if (CoreUtils.isNullOrEmpty(uri.getQuery())) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("URI does not contain a query component. FileSystems require a URI of " + + "the format \"azb://?endpoint=\".")); + } + + String endpoint = Flux.fromArray(uri.getQuery().split("&")) + .filter(s -> s.startsWith(ENDPOINT_QUERY_KEY + "=")) + .switchIfEmpty(Mono.defer(() -> Mono.error(LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("URI does not contain an \"" + ENDPOINT_QUERY_KEY + "=\" parameter. " + + "FileSystems require a URI of the format \"azb://?endpoint=\""))))) + .map(s -> s.substring(ENDPOINT_QUERY_KEY.length() + 1)) // Trim the query key and = + .blockLast(); + + if (CoreUtils.isNullOrEmpty(endpoint)) { + throw LoggingUtility.logError(ClientLoggerHolder.LOGGER, + new IllegalArgumentException("No account endpoint provided in URI query.")); + } + + return endpoint; + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzurePath.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzurePath.java new file mode 100644 index 00000000000..917895ba39e --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzurePath.java @@ -0,0 +1,836 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import java.io.File; +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.ClosedFileSystemException; +import java.nio.file.FileSystem; +import java.nio.file.InvalidPathException; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.WatchEvent; +import java.nio.file.WatchKey; +import java.nio.file.WatchService; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Deque; +import java.util.Iterator; +import java.util.List; +import java.util.Objects; +import java.util.stream.Stream; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobUrlParts; + +/** + * An object that may be used to locate a file in a file system. + *

+ * The root component, if it is present, is the first element of the path and is denoted by a {@code ':'} as the last + * character. Hence, only one instance of {@code ':'} may appear in a path string, and it may only be the last character + * of the first element in the path. The root component is used to identify which container a path belongs to. All other + * path elements, including separators, are considered as the blob name. {@link AzurePath#fromBlobUrl} may + * be used to convert a typical http url pointing to a blob into an {@code AzurePath} object pointing to the same + * resource. + *

+ * Constructing a syntactically valid path does not ensure a resource exists at the given path. An error will + * not be thrown until trying to access an invalid resource, e.g. trying to access a resource that does not exist. + *

+ * Path names are case-sensitive. + *

+ * If a resource is accessed via a relative path, it will be resolved against the default directory of the file system. + * The default directory is as defined in the {@link AzureFileSystem} docs. + *

+ * Leading and trailing separators will be stripped from each component passed to + * {@link AzureFileSystem#getPath(String, String...)}. This has the effect of treating "foo/" as though it were simply + * "foo". + */ +public final class AzurePath implements Path { + private static final ClientLogger LOGGER = new ClientLogger(AzurePath.class); + static final String ROOT_DIR_SUFFIX = ":"; + + private final AzureFileSystem parentFileSystem; + private final String pathString; + + AzurePath(AzureFileSystem parentFileSystem, String first, String... more) { + this.parentFileSystem = parentFileSystem; + + /* + Break all strings into their respective elements and remove empty elements. This has the effect of stripping + any trailing, leading, or internal delimiters so there are no duplicates/empty elements when we join. + */ + List elements = new ArrayList<>(Arrays.asList(first.split(parentFileSystem.getSeparator()))); + if (more != null) { + for (String next : more) { + elements.addAll(Arrays.asList(next.split(parentFileSystem.getSeparator()))); + } + } + elements.removeIf(String::isEmpty); + + this.pathString = String.join(this.parentFileSystem.getSeparator(), elements); + + // Validate the path string by checking usage of the reserved character ROOT_DIR_SUFFIX. + for (int i = 0; i < elements.size(); i++) { + String element = elements.get(i); + /* + If there is a root component, it must be the first element. A root component takes the format of + ":". The ':', or ROOT_DIR_SUFFIX, if present, can only appear once, and can only be the last + character of the first element. + */ + if (i == 0) { + if (element.contains(ROOT_DIR_SUFFIX) && element.indexOf(ROOT_DIR_SUFFIX) < element.length() - 1) { + throw LoggingUtility.logError(LOGGER, new InvalidPathException(this.pathString, ROOT_DIR_SUFFIX + + " may only be used as the last character in the root component of a path")); + } + // No element besides the first may contain the ROOT_DIR_SUFFIX, as only the first element may be the root. + } else if (element.contains(ROOT_DIR_SUFFIX)) { + throw LoggingUtility.logError(LOGGER, new InvalidPathException(this.pathString, ROOT_DIR_SUFFIX + + " is an invalid character except to identify the root element of this path if there is one.")); + } + } + } + + /** + * Returns the file system that created this object. + * + * @return the file system that created this object + */ + @Override + public FileSystem getFileSystem() { + return this.parentFileSystem; + } + + /** + * Tells whether this path is absolute. + *

+ * An absolute path is complete in that it doesn't need to be combined with other path information in order to + * locate a file. A path is considered absolute in this file system if it contains a root component. + * + * @return whether the path is absolute + */ + @Override + public boolean isAbsolute() { + return this.getRoot() != null; + } + + /** + * Returns the root component of this path as a Path object, or null if this path does not have a root component. + *

+ * The root component of this path also identifies the Azure Storage Container in which the file is stored. This + * method will not validate that the root component corresponds to an actual file store/container in this + * file system. It will simply return the root component of the path if one is present and syntactically valid. + * + * @return a path representing the root component of this path, or null + */ + @Override + public Path getRoot() { + // Check if the first element of the path is formatted like a root directory. + String[] elements = this.splitToElements(); + if (elements.length > 0 && elements[0].endsWith(ROOT_DIR_SUFFIX)) { + return this.parentFileSystem.getPath(elements[0]); + } + return null; + } + + /** + * Returns the name of the file or directory denoted by this path as a Path object. The file name is the farthest + * element from the root in the directory hierarchy. + * + * @return a path representing the name of the file or directory, or null if this path has zero elements + */ + @Override + public Path getFileName() { + if (this.isRoot()) { + return null; + } else if (this.pathString.isEmpty()) { + return this; + } else { + List elements = Arrays.asList(this.splitToElements()); + return this.parentFileSystem.getPath(elements.get(elements.size() - 1)); + } + } + + /** + * Returns the parent path, or null if this path does not have a parent. + *

+ * The parent of this path object consists of this path's root component, if any, and each element in the path + * except for the farthest from the root in the directory hierarchy. This method does not access the file system; + * the path or its parent may not exist. Furthermore, this method does not eliminate special names such as "." and + * ".." that may be used in some implementations. On UNIX for example, the parent of "/a/b/c" is "/a/b", and the + * parent of "x/y/." is "x/y". This method may be used with the normalize method, to eliminate redundant names, for + * cases where shell-like navigation is required. + *

+ * If this path has one or more elements, and no root component, then this method is equivalent to evaluating the + * expression: + * + * {@code subpath(0, getNameCount()-1);} + * + * @return a path representing the path's parent + */ + @Override + public Path getParent() { + /* + If this path only has one element or is empty, there is no parent. Note the root is included in the parent, so + we don't use getNameCount here. + */ + String[] elements = this.splitToElements(); + if (elements.length == 1 || elements.length == 0) { + return null; + } + + return this.parentFileSystem.getPath( + this.pathString.substring(0, this.pathString.lastIndexOf(this.parentFileSystem.getSeparator()))); + } + + /** + * Returns the number of name elements in the path. + * + * @return the number of elements in the path, or 0 if this path only represents a root component + */ + @Override + public int getNameCount() { + if (this.pathString.isEmpty()) { + return 1; + } + return this.splitToElements(this.withoutRoot()).length; + } + + /** + * Returns a name element of this path as a Path object. + *

+ * The index parameter is the index of the name element to return. The element that is closest to the root in the + * directory hierarchy has index 0. The element that is farthest from the root has index {@code count-1}. + * + * @param index the index of the element + * @return the name element + * @throws IllegalArgumentException if index is negative, index is greater than or equal to the number of elements, + * or this path has zero name elements + */ + @Override + public Path getName(int index) { + if (index < 0 || index >= this.getNameCount()) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException(String.format("Index %d is out of " + + "bounds", index))); + } + // If the path is empty, the only valid option is also an empty path. + if (this.pathString.isEmpty()) { + return this; + } + + return this.parentFileSystem.getPath(this.splitToElements(this.withoutRoot())[index]); + } + + /** + * Returns a relative Path that is a subsequence of the name elements of this path. + *

+ * The beginIndex and endIndex parameters specify the subsequence of name elements. The name that is closest to the + * root in the directory hierarchy has index 0. The name that is farthest from the root has index {@code count-1}. + * The returned Path object has the name elements that begin at beginIndex and extend to the element at index + * {@code endIndex-1}. + * + * @param begin the index of the first element, inclusive + * @param end the index of the last element, exclusive + * @return a new Path object that is a subsequence of the name elements in this Path + */ + @Override + public Path subpath(int begin, int end) { + if (begin < 0 || begin >= this.getNameCount() + || end <= begin || end > this.getNameCount()) { + throw LoggingUtility.logError(LOGGER, + new IllegalArgumentException(String.format("Values of begin: %d and end: %d are invalid", begin, end))); + } + + String[] subnames = Stream.of(this.splitToElements(this.withoutRoot())) + .skip(begin) + .limit(end - begin) + .toArray(String[]::new); + + return this.parentFileSystem.getPath(String.join(this.parentFileSystem.getSeparator(), subnames)); + } + + /** + * Tests if this path starts with the given path. + *

+ * This path starts with the given path if this path's root component starts with the root component of the given + * path, and this path starts with the same name elements as the given path. If the given path has more name + * elements than this path then false is returned. + *

+ * If this path does not have a root component and the given path has a root component then this path does not start + * with the given path. + *

+ * If the given path is associated with a different FileSystem to this path then false is returned. + *

+ * In this implementation, a root component starts with another root component if the two root components are + * equivalent strings. In other words, if the files are stored in the same container. + * + * @param path the given path + * @return true if this path starts with the given path; otherwise false + */ + @Override + public boolean startsWith(Path path) { + if (!path.getFileSystem().equals(this.parentFileSystem)) { + return false; + } + + // An empty path never starts with another path and is never the start of another path. + if (this.pathString.isEmpty() ^ ((AzurePath) path).pathString.isEmpty()) { + return false; + } + + String[] thisPathElements = this.splitToElements(); + String[] otherPathElements = ((AzurePath) path).splitToElements(); + if (otherPathElements.length > thisPathElements.length) { + return false; + } + for (int i = 0; i < otherPathElements.length; i++) { + if (!otherPathElements[i].equals(thisPathElements[i])) { + return false; + } + } + + return true; + } + + /** + * Tests if this path starts with a Path, constructed by converting the given path string, in exactly the manner + * specified by the startsWith(Path) method. + * + * @param path the given path string + * @return true if this path starts with the given path; otherwise false + * @throws InvalidPathException If the path string cannot be converted to a Path. + */ + @Override + public boolean startsWith(String path) { + return this.startsWith(this.parentFileSystem.getPath(path)); + } + + /** + * Tests if this path ends with the given path. + *

+ * If the given path has N elements, and no root component, and this path has N or more elements, then this path + * ends with the given path if the last N elements of each path, starting at the element farthest from the root, + * are equal. + *

+ * If the given path has a root component then this path ends with the given path if the root component of this path + * ends with the root component of the given path, and the corresponding elements of both paths are equal. If this + * path does not have a root component and the given path has a root component then this path does not end with the + * given path. + *

+ * If the given path is associated with a different FileSystem to this path then false is returned. + *

+ * In this implementation, a root component ends with another root component if the two root components are + * equivalent strings. In other words, if the files are stored in the same container. + * + * @param path the given path + * @return true if this path ends with the given path; otherwise false + */ + @Override + public boolean endsWith(Path path) { + /* + There can only be one instance of a file system with a given id, so comparing object identity is equivalent + to checking ids here. + */ + if (path.getFileSystem() != this.parentFileSystem) { + return false; + } + + // An empty path never ends with another path and is never the end of another path. + if (this.pathString.isEmpty() ^ ((AzurePath) path).pathString.isEmpty()) { + return false; + } + + String[] thisPathElements = this.splitToElements(); + String[] otherPathElements = ((AzurePath) path).splitToElements(); + if (otherPathElements.length > thisPathElements.length) { + return false; + } + // If the given path has a root component, the paths must be equal. + if (path.getRoot() != null && otherPathElements.length != thisPathElements.length) { + return false; + } + for (int i = 1; i <= otherPathElements.length; i++) { + if (!otherPathElements[otherPathElements.length - i] + .equals(thisPathElements[thisPathElements.length - i])) { + return false; + } + } + return true; + } + + /** + * Tests if this path ends with a Path, constructed by converting the given path string, in exactly the manner + * specified by the endsWith(Path) method. + * + * @param path the given path string + * @return true if this path starts with the given path; otherwise false + * @throws InvalidPathException If the path string cannot be converted to a Path. + */ + @Override + public boolean endsWith(String path) { + return this.endsWith(this.parentFileSystem.getPath(path)); + } + + /** + * Returns a path that is this path with redundant name elements eliminated. + *

+ * It derives from this path, a path that does not contain redundant name elements. The "." and ".." are special + * names used to indicate the current directory and parent directory. All occurrences of "." are considered + * redundant. If a ".." is preceded by a non-".." name then both names are considered redundant (the process to + * identify such names is repeated until is it no longer applicable). + *

+ * This method does not access the file system; the path may not locate a file that exists. Eliminating ".." and a + * preceding name from a path may result in the path that locates a different file than the original path + * + * @return the resulting path or this path if it does not contain redundant name elements; an empty path is returned + * if this path does have a root component and all name elements are redundant + * + */ + @Override + public Path normalize() { + Deque stack = new ArrayDeque<>(); + String[] pathElements = this.splitToElements(); + Path root = this.getRoot(); + String rootStr = root == null ? null : root.toString(); + for (String element : pathElements) { + if (".".equals(element)) { + continue; + } else if ("..".equals(element)) { + if (rootStr != null) { + // Root path. We never push "..". + if (!stack.isEmpty() && stack.peekLast().equals(rootStr)) { + // Cannot go higher than root. Ignore. + continue; + } else { + stack.removeLast(); + } + } else { + // Relative paths can have an arbitrary number of ".." at the beginning. + if (stack.isEmpty()) { + stack.addLast(element); + } else if (stack.peek().equals("..")) { + stack.addLast(element); + } else { + stack.removeLast(); + } + } + } else { + stack.addLast(element); + } + } + + return this.parentFileSystem.getPath("", stack.toArray(new String[0])); + } + + /** + * Resolve the given path against this path. + *

+ * If the other parameter is an absolute path then this method trivially returns other. If other is an empty path + * then this method trivially returns this path. Otherwise, this method considers this path to be a directory and + * resolves the given path against this path. In the simplest case, the given path does not have a root component, + * in which case this method joins the given path to this path and returns a resulting path that ends with the given + * path. Where the given path has a root component then resolution is highly implementation dependent and therefore + * unspecified. + * + * @param path the path to resolve against this path + * @return the resulting path + */ + @Override + public Path resolve(Path path) { + if (path.isAbsolute()) { + return path; + } + if (path.getNameCount() == 0) { + return this; + } + return this.parentFileSystem.getPath(this.toString(), path.toString()); + } + + /** + * Converts a given path string to a Path and resolves it against this Path in exactly the manner specified by the + * {@link #resolve(Path) resolve} method. + * + * @param path the path string to resolve against this path + * @return the resulting path + * @throws InvalidPathException if the path string cannot be converted to a Path. + */ + @Override + public Path resolve(String path) { + return this.resolve(this.parentFileSystem.getPath(path)); + } + + /** + * Resolves the given path against this path's parent path. This is useful where a file name needs to be replaced + * with another file name. For example, suppose that the name separator is "/" and a path represents + * "dir1/dir2/foo", then invoking this method with the Path "bar" will result in the Path "dir1/dir2/bar". If this + * path does not have a parent path, or other is absolute, then this method returns other. If other is an empty path + * then this method returns this path's parent, or where this path doesn't have a parent, the empty path. + * + * @param path the path to resolve against this path's parent + * @return the resulting path + */ + @Override + public Path resolveSibling(Path path) { + if (path.isAbsolute()) { + return path; + } + + Path parent = this.getParent(); + return parent == null ? path : parent.resolve(path); + } + + /** + * Converts a given path string to a Path and resolves it against this path's parent path in exactly the manner + * specified by the resolveSibling method. + * + * @param path the path string to resolve against this path's parent + * @return the resulting path + * @throws InvalidPathException if the path string cannot be converted to a Path. + */ + @Override + public Path resolveSibling(String path) { + return this.resolveSibling(this.parentFileSystem.getPath(path)); + } + + /** + * Constructs a relative path between this path and a given path. + *

+ * Relativization is the inverse of resolution. This method attempts to construct a relative path that when resolved + * against this path, yields a path that locates the same file as the given path. + *

+ * A relative path cannot be constructed if only one of the paths have a root component. If both paths have a root + * component, it is still possible to relativize one against the other. If this path and the given path are equal + * then an empty path is returned. + *

+ * For any two normalized paths p and q, where q does not have a root component, + * {@code p.relativize(p.resolve(q)).equals(q)} + * + * @param path the path to relativize against this path + * @return the resulting relative path, or an empty path if both paths are equal + * @throws IllegalArgumentException if other is not a Path that can be relativized against this path + */ + @Override + public Path relativize(Path path) { + if (path.getRoot() == null ^ this.getRoot() == null) { + throw LoggingUtility.logError(LOGGER, + new IllegalArgumentException("Both paths must be absolute or neither can be")); + } + + AzurePath thisNormalized = (AzurePath) this.normalize(); + Path otherNormalized = path.normalize(); + + Deque deque = new ArrayDeque<>( + Arrays.asList(otherNormalized.toString().split(this.parentFileSystem.getSeparator()))); + + int i = 0; + String[] thisElements = thisNormalized.splitToElements(); + while (i < thisElements.length && !deque.isEmpty() && thisElements[i].equals(deque.peekFirst())) { + deque.removeFirst(); + i++; + } + while (i < thisElements.length) { + deque.addFirst(".."); + i++; + } + + return this.parentFileSystem.getPath("", deque.toArray(new String[0])); + } + + /** + * Returns a URI to represent this path. + *

+ * This method constructs an absolute URI with a scheme equal to the URI scheme that identifies the provider. + *

+ * No authority component is defined for the {@code URI} returned by this method. This implementation offers the + * same equivalence guarantee as the default provider. + * + * @return the URI representing this path + * @throws SecurityException never + */ + @Override + public URI toUri() { + try { + return new URI(this.parentFileSystem.provider().getScheme(), null, "/" + this.toAbsolutePath(), + null, null); + } catch (URISyntaxException e) { + throw LoggingUtility.logError(LOGGER, new IllegalStateException("Unable to create valid URI from path", e)); + } + } + + /** + * Returns a Path object representing the absolute path of this path. + *

+ * If this path is already absolute then this method simply returns this path. Otherwise, this method resolves the + * path against the default directory. + * + * @return a Path object representing the absolute path + * @throws SecurityException never + */ + @Override + public Path toAbsolutePath() { + if (this.isAbsolute()) { + return this; + } + return this.parentFileSystem.getDefaultDirectory().resolve(this); + } + + /** + * Unsupported. + * + * @param linkOptions options + * @return the real path + * @throws UnsupportedOperationException operation not supported. + */ + @Override + public Path toRealPath(LinkOption... linkOptions) throws IOException { + throw new UnsupportedOperationException("Symbolic links are not supported."); + } + + /** + * Unsupported. + * + * @return the file + * @throws UnsupportedOperationException operation not supported. + */ + @Override + public File toFile() { + throw new UnsupportedOperationException(); + } + + /** + * Unsupported. + * + * @param watchService watchService + * @param kinds kinds + * @param modifiers modifiers + * @return the watch key + * @throws UnsupportedOperationException operation not supported. + */ + @Override + public WatchKey register(WatchService watchService, WatchEvent.Kind[] kinds, WatchEvent.Modifier... modifiers) + throws IOException { + throw new UnsupportedOperationException("WatchEvents are not supported."); + } + + /** + * Unsupported. + * + * @param watchService watchService + * @param kinds kinds + * @return the watch key + * @throws UnsupportedOperationException operation not supported. + */ + @Override + public WatchKey register(WatchService watchService, WatchEvent.Kind... kinds) throws IOException { + throw new UnsupportedOperationException("WatchEvents are not supported."); + } + + /** + * Returns an iterator over the name elements of this path. + *

+ * The first element returned by the iterator represents the name element that is closest to the root in the + * directory hierarchy, the second element is the next closest, and so on. The last element returned is the name of + * the file or directory denoted by this path. The root component, if present, is not returned by the iterator. + * + * @return an iterator over the name elements of this path. + */ + @Override + public Iterator iterator() { + if (this.pathString.isEmpty()) { + return Collections.singletonList((Path) this).iterator(); + } + return Arrays.asList(Stream.of(this.splitToElements(this.withoutRoot())) + .map(s -> this.parentFileSystem.getPath(s)) + .toArray(Path[]::new)) + .iterator(); + } + + /** + * Compares two abstract paths lexicographically. This method does not access the file system and neither file is + * required to exist. + *

+ * This method may not be used to compare paths that are associated with different file system providers. + *

+ * This result of this method is identical to a string comparison on the underlying path strings. + * + * @return zero if the argument is equal to this path, a value less than zero if this path is lexicographically less + * than the argument, or a value greater than zero if this path is lexicographically greater than the argument + * @throws ClassCastException if the paths are associated with different providers + */ + @Override + public int compareTo(Path path) { + if (!(path instanceof AzurePath)) { + throw LoggingUtility.logError(LOGGER, new ClassCastException("Other path is not an instance of " + + "AzurePath.")); + } + + return this.pathString.compareTo(((AzurePath) path).pathString); + } + + /** + * Returns the string representation of this path. + *

+ * If this path was created by converting a path string using the getPath method then the path string returned by + * this method may differ from the original String used to create the path. + *

+ * The returned path string uses the default name separator to separate names in the path. + * + * @return the string representation of this path + */ + @Override + public String toString() { + return this.pathString; + } + + /** + * A path is considered equal to another path if it is associated with the same file system instance and if the + * path strings are equivalent. + * + * @return true if, and only if, the given object is a Path that is identical to this Path + */ + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + AzurePath paths = (AzurePath) o; + return Objects.equals(parentFileSystem, paths.parentFileSystem) + && Objects.equals(pathString, paths.pathString); + } + + @Override + public int hashCode() { + return Objects.hash(parentFileSystem, pathString); + } + + /** + * Returns a {@link BlobClient} which references a blob pointed to by this path. Note that this does not guarantee + * the existence of the blob at this location. + * + * @return a {@link BlobClient}. + * @throws IOException If the path only contains a root component or is empty + */ + public BlobClient toBlobClient() throws IOException { + /* + We don't store the blob client because unlike other types in this package, a Path does not actually indicate the + existence or even validity of any remote resource. It is purely a representation of a path. Therefore, we do not + construct the client or perform any validation until it is requested. + */ + // Converting to an absolute path ensures there is a container to operate on even if it is the default. + // Normalizing ensures the path is clean. + Path root = this.normalize().toAbsolutePath().getRoot(); + if (root == null) { + throw LoggingUtility.logError(LOGGER, + new IllegalStateException("Root should never be null after calling toAbsolutePath.")); + } + String fileStoreName = this.rootToFileStore(root.toString()); + + BlobContainerClient containerClient = + ((AzureFileStore) this.parentFileSystem.getFileStore()).getContainerClient(); + + String blobName = this.withoutRoot(); + if (blobName.isEmpty()) { + throw LoggingUtility.logError(LOGGER, new IOException("Cannot get a blob client to a path that only " + + "contains the root or is an empty path")); + } + + return containerClient.getBlobClient(blobName); + } + + /** + * A utility method to conveniently convert from a URL to a storage resource to an {@code AzurePath} pointing to the + * same resource. + * + * The url must be well formatted. There must be an open filesystem corresponding to the account which contains the + * blob. Otherwise, a {@link java.nio.file.FileSystemNotFoundException} will be thrown. + * + * The url may point to either an account, container, or blob. If it points to an account, the path will be empty, + * but it will have an internal reference to the file system containing it, meaning instance methods may be + * performed on the path to construct a reference to another object. If it points to a container, there will be one + * element, which is the root element. Everything after the container, that is the blob name, will then be appended + * after the root element. + * + * IP style urls are not currently supported. + * + * The {@link AzureFileSystemProvider} can typically be obtained via {@link AzureFileSystem#provider()}. + * + * @param provider The installed {@link AzureFileSystemProvider} that manages open file systems for this jvm. + * @param url The url to the desired resource. + * @return An {@link AzurePath} which points to the resource identified by the url. + * @throws URISyntaxException If the url contains elements which are not well formatted. + */ + public static AzurePath fromBlobUrl(AzureFileSystemProvider provider, String url) throws URISyntaxException { + BlobUrlParts parts = BlobUrlParts.parse(url); + URI fileSystemUri = hostToFileSystemUri(provider, parts.getScheme(), parts.getHost()); + FileSystem parentFileSystem = provider.getFileSystem(fileSystemUri); + return new AzurePath((AzureFileSystem) parentFileSystem, fileStoreToRoot(parts.getBlobContainerName()), + parts.getBlobName() == null ? "" : parts.getBlobName()); + } + + /** + * @return Whether this path consists of only a root component. + */ + boolean isRoot() { + return this.equals(this.getRoot()); + } + + private String withoutRoot() { + Path root = this.getRoot(); + String str = this.pathString; + if (root != null) { + str = this.pathString.substring(root.toString().length()); + } + if (str.startsWith(this.parentFileSystem.getSeparator())) { + str = str.substring(1); + } + + return str; + } + + private String[] splitToElements() { + return this.splitToElements(this.pathString); + } + + private String[] splitToElements(String str) { + String[] arr = str.split(this.parentFileSystem.getSeparator()); + /* + This is a special case where we split after removing the root from a path that is just the root. Or otherwise + have an empty path. + */ + if (arr.length == 1 && arr[0].isEmpty()) { + return new String[0]; + } + return arr; + } + + private String rootToFileStore(String root) { + return root.substring(0, root.length() - 1); // Remove the ROOT_DIR_SUFFIX + } + + private static String fileStoreToRoot(String fileStore) { + if (fileStore == null || "".equals(fileStore)) { + return ""; + } + return fileStore + ROOT_DIR_SUFFIX; + } + + private static URI hostToFileSystemUri(AzureFileSystemProvider provider, String scheme, String host) + throws URISyntaxException { + return new URI(provider.getScheme() + "://?endpoint=" + scheme + "://" + host); + } + + static void ensureFileSystemOpen(Path p) { + if (!p.getFileSystem().isOpen()) { + throw LoggingUtility.logError(LOGGER, new ClosedFileSystemException()); + } + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureResource.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureResource.java new file mode 100644 index 00000000000..92fb14a62cc --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureResource.java @@ -0,0 +1,284 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.BlobContainerClient; +import com.azure.storage.blob.BlobClient; +import com.azure.storage.blob.BlobContainerClientBuilder; +import com.azure.storage.blob.models.BlobHttpHeaders; +import com.azure.storage.blob.models.BlobItem; +import com.azure.storage.blob.models.BlobListDetails; +import com.azure.storage.blob.models.BlobProperties; +import com.azure.storage.blob.models.BlobRequestConditions; +import com.azure.storage.blob.models.BlobStorageException; +import com.azure.storage.blob.models.ListBlobsOptions; +import com.azure.storage.blob.models.ParallelTransferOptions; +import com.azure.storage.blob.options.BlockBlobOutputStreamOptions; +import com.azure.storage.blob.specialized.BlobOutputStream; +import com.azure.storage.common.implementation.Constants; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.attribute.FileAttribute; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * This type is meant to be a logical grouping of operations and data associated with an azure resource. It is NOT + * intended to serve as a local cache for any data related to remote resources. It is agnostic to whether the resource + * is a directory or a file and will not perform any validation of the resource type, though root directories are not + * supported as they are backed by containers and do not support many file system apis. + * + * It also serves as the interface to Storage clients. Any operation that needs to use a client should first build an + * AzureResource using a path and then use the getter to access the client. + */ +final class AzureResource { + private static final ClientLogger LOGGER = new ClientLogger(AzureResource.class); + + static final String DIR_METADATA_MARKER = Constants.HeaderConstants.DIRECTORY_METADATA_KEY; + + private final AzurePath path; + private final BlobClient blobClient; + + // The following are not kept consistent with the service. They are only held here between parsing and putting. + private BlobHttpHeaders blobHeaders; + private Map blobMetadata; + + AzureResource(Path path) throws IOException { + Objects.requireNonNull(path, "path"); + this.path = validatePathInstanceType(path); + this.validateNotRoot(); + this.blobClient = this.path.toBlobClient(); + } + + /** + * Checks for the existence of the parent of the given path. We do not check for the actual marker blob as parents + * need only weakly exist. + * + * If the parent is a root (container), it will be assumed to exist, so it must be validated elsewhere that the + * container is a legitimate root within this file system. + */ + boolean checkParentDirectoryExists() throws IOException { + /* + If the parent is just the root (or null, which means the parent is implicitly the default directory which is a + root), that means we are checking a container, which is always considered to exist. Otherwise, perform normal + existence check. + */ + Path parent = this.path.getParent(); + return (parent == null || parent.equals(path.getRoot())) + || new AzureResource(this.path.getParent()).checkDirectoryExists(); + } + + /** + * Checks whether a directory exists by either being empty or having children. + */ + boolean checkDirectoryExists() throws IOException { + DirectoryStatus dirStatus = this.checkDirStatus(); + return dirStatus.equals(DirectoryStatus.EMPTY) || dirStatus.equals(DirectoryStatus.NOT_EMPTY); + } + + /* + This method will check specifically whether there is a virtual directory at this location. It must be known before + that there is no file present at the destination. + */ + boolean checkVirtualDirectoryExists() throws IOException { + DirectoryStatus dirStatus = this.checkDirStatus(false); + return dirStatus.equals(DirectoryStatus.NOT_EMPTY); // Virtual directories cannot be empty + } + + /** + * This method will check if a directory is extant and/or empty and accommodates virtual directories. This method + * will not check the status of root directories. + */ + DirectoryStatus checkDirStatus() throws IOException { + if (this.blobClient == null) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("The blob client was null.")); + } + + /* + * Do a get properties first on the directory name. This will determine if it is concrete&&exists or is either + * virtual or doesn't exist. + */ + BlobProperties props = null; + boolean exists = false; + try { + props = this.getBlobClient().getProperties(); + exists = true; + } catch (BlobStorageException e) { + if (e.getStatusCode() != 404) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + // Check if the resource is a file or directory before listing + if (exists && !props.getMetadata().containsKey(AzureResource.DIR_METADATA_MARKER)) { + return DirectoryStatus.NOT_A_DIRECTORY; + } + + return checkDirStatus(exists); + } + + /* + This method will determine the status of the directory given it is already known whether or not there is an object + at the target. + */ + DirectoryStatus checkDirStatus(boolean exists) throws IOException { + BlobContainerClient containerClient = this.getContainerClient(); + + // List on the directory name + '/' so that we only get things under the directory if any + ListBlobsOptions listOptions = new ListBlobsOptions().setMaxResultsPerPage(2) + .setPrefix(this.blobClient.getBlobName() + AzureFileSystem.PATH_SEPARATOR) + .setDetails(new BlobListDetails().setRetrieveMetadata(true)); + + /* + * If listing returns anything, then it is not empty. If listing returns nothing and exists() was true, then it's + * empty Else it does not exist + */ + try { + Iterator blobIterator = containerClient.listBlobsByHierarchy(AzureFileSystem.PATH_SEPARATOR, + listOptions, null).iterator(); + if (blobIterator.hasNext()) { + return DirectoryStatus.NOT_EMPTY; + } else if (exists) { + return DirectoryStatus.EMPTY; + } else { + return DirectoryStatus.DOES_NOT_EXIST; + } + } catch (BlobStorageException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + /** + * Creates the actual directory marker. This method should only be used when any necessary checks for proper + * conditions of directory creation (e.g. parent existence) have already been performed. Otherwise, + * {@link AzureFileSystemProvider#createDirectory(Path, FileAttribute[])} should be preferred. + * + * @param requestConditions Any necessary request conditions to pass when creating the directory blob. + */ + void putDirectoryBlob(BlobRequestConditions requestConditions) { + this.blobClient.getBlockBlobClient().commitBlockListWithResponse(Collections.emptyList(), this.blobHeaders, + this.prepareMetadataForDirectory(), null, requestConditions, null, null); + } + + /* + Note that this will remove the properties from the list of attributes as it finds them. + */ + private void extractHttpHeaders(List> fileAttributes) { + BlobHttpHeaders headers = new BlobHttpHeaders(); + for (Iterator> it = fileAttributes.iterator(); it.hasNext();) { + FileAttribute attr = it.next(); + boolean propertyFound = true; + switch (attr.name()) { + case AzureFileSystemProvider.CONTENT_TYPE: + headers.setContentType(attr.value().toString()); + break; + case AzureFileSystemProvider.CONTENT_LANGUAGE: + headers.setContentLanguage(attr.value().toString()); + break; + case AzureFileSystemProvider.CONTENT_DISPOSITION: + headers.setContentDisposition(attr.value().toString()); + break; + case AzureFileSystemProvider.CONTENT_ENCODING: + headers.setContentEncoding(attr.value().toString()); + break; + case AzureFileSystemProvider.CONTENT_MD5: + if ((attr.value() instanceof byte[])) { + headers.setContentMd5((byte[]) attr.value()); + } else { + throw LoggingUtility.logError(LOGGER, + new UnsupportedOperationException("Content-MD5 attribute must be a byte[]")); + } + break; + case AzureFileSystemProvider.CACHE_CONTROL: + headers.setCacheControl(attr.value().toString()); + break; + default: + propertyFound = false; + break; + } + + if (propertyFound) { + it.remove(); + } + } + + this.blobHeaders = headers; + } + + /** + * Note this should only be used after the headers have been extracted. + * + * @param fileAttributes The attributes to convert to metadata. + */ + private void convertAttributesToMetadata(List> fileAttributes) { + Map metadata = new HashMap<>(); + for (FileAttribute attr : fileAttributes) { + metadata.put(attr.name(), attr.value().toString()); + } + + // If no attributes are set, return null so existing metadata is not cleared. + this.blobMetadata = metadata.isEmpty() ? null : metadata; + } + + private void validateNotRoot() { + if (this.path.isRoot()) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException( + "Root directory not supported. Path: " + this.path)); + } + } + + private AzurePath validatePathInstanceType(Path path) { + if (!(path instanceof AzurePath)) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("This provider cannot operate on " + + "subtypes of Path other than AzurePath")); + } + return (AzurePath) path; + } + + BlobContainerClient getContainerClient() { + return new BlobContainerClientBuilder().endpoint(this.blobClient.getBlobUrl()) + .pipeline(this.blobClient.getHttpPipeline()) + .buildClient(); + } + + AzureResource setFileAttributes(List> attributes) { + attributes = new ArrayList<>(attributes); // To ensure removing header values from the list is supported. + extractHttpHeaders(attributes); + convertAttributesToMetadata(attributes); + + return this; + } + + AzurePath getPath() { + return this.path; + } + + BlobClient getBlobClient() { + return this.blobClient; + } + + BlobOutputStream getBlobOutputStream(ParallelTransferOptions pto, BlobRequestConditions rq) { + BlockBlobOutputStreamOptions options = new BlockBlobOutputStreamOptions() + .setHeaders(this.blobHeaders) + .setMetadata(this.blobMetadata) + .setParallelTransferOptions(pto) + .setRequestConditions(rq); + return this.blobClient.getBlockBlobClient().getBlobOutputStream(options); + } + + private Map prepareMetadataForDirectory() { + if (this.blobMetadata == null) { + this.blobMetadata = new HashMap<>(); + } + this.blobMetadata.put(DIR_METADATA_MARKER, "true"); + return this.blobMetadata; + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureSeekableByteChannel.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureSeekableByteChannel.java new file mode 100644 index 00000000000..e51e727450b --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/AzureSeekableByteChannel.java @@ -0,0 +1,245 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.ClosedChannelException; +import java.nio.channels.NonReadableChannelException; +import java.nio.channels.NonWritableChannelException; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Path; + +/** + * A byte channel that maintains a current position. + *

+ * A channel may only be opened in read mode OR write mode. It may not be opened in read/write mode. Seeking is + * supported for reads, but not for writes. Modifications to existing files is not permitted--only creating new files or + * overwriting existing files. + *

+ * This type is not threadsafe to prevent having to hold locks across network calls. + */ +public final class AzureSeekableByteChannel implements SeekableByteChannel { + private static final ClientLogger LOGGER = new ClientLogger(AzureSeekableByteChannel.class); + + private final NioBlobInputStream reader; + private final NioBlobOutputStream writer; + private long position; + private boolean closed = false; + private final Path path; + /* + If this type needs to be made threadsafe, closed should be volatile. We need to add a lock to guard updates to + position or make it an atomicLong. If we have a lock, we have to be careful about holding while doing io ops and at + least ensure timeouts are set. We probably have to duplicate or copy the buffers for at least writing to ensure they + don't get overwritten. + */ + + AzureSeekableByteChannel(NioBlobInputStream inputStream, Path path) { + this.reader = inputStream; + /* + We mark at the beginning (we always construct a stream to the beginning of the blob) to support seeking. We can + effectively seek anywhere by always marking at the beginning of the blob and then a seek is resetting to that + mark and skipping. + */ + inputStream.mark(Integer.MAX_VALUE); + this.writer = null; + this.position = 0; + this.path = path; + } + + AzureSeekableByteChannel(NioBlobOutputStream outputStream, Path path) { + this.writer = outputStream; + this.reader = null; + this.position = 0; + this.path = path; + } + + @Override + public int read(ByteBuffer dst) throws IOException { + AzurePath.ensureFileSystemOpen(this.path); + validateOpen(); + validateReadMode(); + + // See comments in position(), remember that position is 0-based and size() is exclusive + if (this.position >= this.size()) { + return -1; // at or past EOF + } + + // If the buffer is backed by an array, we can write directly to that instead of allocating new memory. + int pos; + final int limit; + final byte[] buf; + if (dst.hasArray()) { + // ByteBuffer has a position and limit that define the bounds of the writeable area, and that + // area can be both smaller than the backing array and might not begin at array index 0. + pos = dst.position(); + limit = pos + dst.remaining(); + buf = dst.array(); + } else { + pos = 0; + limit = dst.remaining(); + buf = new byte[limit]; + } + + while (pos < limit) { + int byteCount = this.reader.read(buf, pos, limit - pos); + if (byteCount == -1) { + break; + } + pos += byteCount; + } + + /* + Either write to the destination if we had to buffer separately or just set the position correctly if we wrote + underneath the buffer + */ + int count; + if (dst.hasArray()) { + count = pos - dst.position(); + dst.position(pos); + } else { + count = pos; // original position was 0 + dst.put(buf, 0, count); + } + + this.position += count; + return count; + } + + @Override + public int write(ByteBuffer src) throws IOException { + AzurePath.ensureFileSystemOpen(this.path); + validateOpen(); + validateWriteMode(); + + final int length = src.remaining(); + this.position += length; + + /* + If the buffer is backed by an array, we can read directly from that instead of allocating new memory. + Set the position correctly if we read from underneath the buffer + */ + int pos; + byte[] buf; + if (src.hasArray()) { + // ByteBuffer has a position and limit that define the bounds of the readable area, and that + // area can be both smaller than the backing array and might not begin at array index 0. + pos = src.position(); + buf = src.array(); + src.position(pos + length); + } else { + pos = 0; + buf = new byte[length]; + src.get(buf); // advances src.position() + } + // Either way, the src.position() and this.position have been updated before we know if this write + // will succeed. (Original behavior.) It may be better to update position(s) only *after* success, + // but then on IOException would we know if there was a partial write, and if so how much? + this.writer.write(buf, pos, length); + return length; + } + + @Override + public long position() throws IOException { + AzurePath.ensureFileSystemOpen(this.path); + validateOpen(); + + return this.position; + } + + @Override + public AzureSeekableByteChannel position(long newPosition) throws IOException { + AzurePath.ensureFileSystemOpen(this.path); + validateOpen(); + validateReadMode(); + + if (newPosition < 0) { + throw LoggingUtility.logError(LOGGER, new IllegalArgumentException("Seek position cannot be negative")); + } + + /* + The javadoc says seeking past the end for reading is legal and that it should indicate the end of the file on + the next read. StorageInputStream doesn't allow this, but we can get around that by modifying the + position variable and skipping the actual read (when read is called next); we'll check in read if we've seeked + past the end and short circuit there as well. + + Because we are in read mode this will always give us the size from properties. + */ + if (newPosition > this.size()) { + this.position = newPosition; + return this; + } + this.reader.reset(); // Because we always mark at the beginning, this will reset us back to the beginning. + this.reader.mark(Integer.MAX_VALUE); + long skipAmount = this.reader.skip(newPosition); + if (skipAmount < newPosition) { + throw new IOException("Could not set desired position"); + } + this.position = newPosition; + + return this; + } + + @Override + public long size() throws IOException { + AzurePath.ensureFileSystemOpen(this.path); + validateOpen(); + + /* + If we are in read mode, the size is the size of the file. + If we are in write mode, the size is the amount of data written so far. + */ + if (reader != null) { + return reader.getBlobInputStream().getProperties().getBlobSize(); + } else { + return position; + } + } + + @Override + public AzureSeekableByteChannel truncate(long size) throws IOException { + throw LoggingUtility.logError(LOGGER, new UnsupportedOperationException()); + } + + @Override + public boolean isOpen() { + AzurePath.ensureFileSystemOpen(this.path); + return !this.closed; + } + + @Override + public void close() throws IOException { + AzurePath.ensureFileSystemOpen(this.path); + if (this.reader != null) { + this.reader.close(); + } else { + this.writer.close(); + } + this.closed = true; + } + + Path getPath() { + return this.path; + } + + private void validateOpen() throws ClosedChannelException { + if (this.closed) { + throw LoggingUtility.logError(LOGGER, new ClosedChannelException()); + } + } + + private void validateReadMode() { + if (this.reader == null) { + throw LoggingUtility.logError(LOGGER, new NonReadableChannelException()); + } + } + + private void validateWriteMode() { + if (this.writer == null) { + throw LoggingUtility.logError(LOGGER, new NonWritableChannelException()); + } + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/DirectoryStatus.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/DirectoryStatus.java new file mode 100644 index 00000000000..8356a7ebeb1 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/DirectoryStatus.java @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +/** + * RESERVED FOR INTERNAL USE. + * + * An enum to indicate the status of a directory. + */ +enum DirectoryStatus { + EMPTY, // The directory at least weakly exists and is empty. + + NOT_EMPTY, // The directory at least weakly exists and has one or more children. + + DOES_NOT_EXIST, // There is no resource at this path. + + NOT_A_DIRECTORY; // A resource exists at this path, but it is not a directory. + + static boolean isDirectory(DirectoryStatus status) { + return EMPTY.equals(status) || NOT_EMPTY.equals(status); + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/LoggingUtility.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/LoggingUtility.java new file mode 100644 index 00000000000..3cd503f98c2 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/LoggingUtility.java @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; + +/** + * Only a minimal Utility class to get around a shortcoming in Core's logging. + */ +final class LoggingUtility { + public static T logError(ClientLogger logger, T e) { + logger.error(e.getMessage()); + return e; + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/NioBlobInputStream.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/NioBlobInputStream.java new file mode 100644 index 00000000000..676972dc93a --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/NioBlobInputStream.java @@ -0,0 +1,211 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.specialized.BlobInputStream; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +/** + * Provides an InputStream to read a file stored as an Azure Blob. + */ +public final class NioBlobInputStream extends InputStream { + private static final ClientLogger LOGGER = new ClientLogger(NioBlobInputStream.class); + + private final BlobInputStream blobInputStream; + private final Path path; + + NioBlobInputStream(BlobInputStream blobInputStream, Path path) { + this.blobInputStream = blobInputStream; + this.path = path; + } + + /** + * Returns an estimate of the number of bytes that can be read (or skipped over) from this input stream without + * blocking by the next invocation of a method for this input stream. The next invocation might be the same thread + * or another thread. A single read or skip of this many bytes will not block, but may read or skip fewer bytes. + * + * @return An int which represents an estimate of the number of bytes that can be read (or skipped + * over) from this input stream without blocking, or 0 when it reaches the end of the input stream. + */ + @Override + public synchronized int available() throws IOException { + AzurePath.ensureFileSystemOpen(path); + return this.blobInputStream.available(); + } + + /** + * Closes this input stream and releases any system resources associated with the stream. + */ + @Override + public synchronized void close() throws IOException { + AzurePath.ensureFileSystemOpen(path); + this.blobInputStream.close(); + } + + /** + * Marks the current position in this input stream. A subsequent call to the reset method repositions this stream at + * the last marked position so that subsequent reads re-read the same bytes. + * + * @param readlimit An int which represents the maximum limit of bytes that can be read before the mark + * position becomes invalid. + */ + @Override + public synchronized void mark(final int readlimit) { + this.blobInputStream.mark(readlimit); + } + + /** + * Tests if this input stream supports the mark and reset methods. + * + * @return Returns {@code true} + */ + @Override + public boolean markSupported() { + return this.blobInputStream.markSupported(); + } + + /** + * Reads the next byte of data from the input stream. The value byte is returned as an int in the range 0 to 255. If + * no byte is available because the end of the stream has been reached, the value -1 is returned. This method blocks + * until input data is available, the end of the stream is detected, or an exception is thrown. + * + * @return An int which represents the total number of bytes read into the buffer, or -1 if there is no + * more data because the end of the stream has been reached. + * @throws IOException If an I/O error occurs. + */ + @Override + public int read() throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + return this.blobInputStream.read(); + /* + BlobInputStream only throws RuntimeException, and it doesn't preserve the cause, it only takes the message, + so we can't do any better than re-wrapping it in an IOException. + */ + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + /** + * Reads some number of bytes from the input stream and stores them into the buffer array b. The number + * of bytes actually read is returned as an integer. This method blocks until input data is available, end of file + * is detected, or an exception is thrown. If the length of b is zero, then no bytes are read and 0 is + * returned; otherwise, there is an attempt to read at least one byte. If no byte is available because the stream is + * at the end of the file, the value -1 is returned; otherwise, at least one byte is read and stored into + * b. + * + * The first byte read is stored into element b[0], the next one into b[1], and so on. The + * number of bytes read is, at most, equal to the length of b. Let k be the number of + * bytes actually read; these bytes will be stored in elements b[0] through b[k-1], + * leaving elements b[k] through + * b[b.length-1] unaffected. + * + * The read(b) method for class {@link InputStream} has the same effect as: + * + * read(b, 0, b.length) + * + * @param b A byte array which represents the buffer into which the data is read. + * @throws IOException If the first byte cannot be read for any reason other than the end of the file, if the input + * stream has been closed, or if some other I/O error occurs. + * @throws NullPointerException If the byte array b is null. + */ + @Override + public int read(final byte[] b) throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + return this.blobInputStream.read(b); + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + /** + * Reads up to len bytes of data from the input stream into an array of bytes. An attempt is made to + * read as many as len bytes, but a smaller number may be read. The number of bytes actually read is + * returned as an integer. This method blocks until input data is available, end of file is detected, or an + * exception is thrown. + * + * If len is zero, then no bytes are read and 0 is returned; otherwise, there is an attempt to read at + * least one byte. If no byte is available because the stream is at end of file, the value -1 is returned; + * otherwise, at least one byte is read and stored into b. + * + * The first byte read is stored into element b[off], the next one into b[off+1], and so + * on. The number of bytes read is, at most, equal to len. Let k be the number of bytes + * actually read; these bytes will be stored in elements b[off] through b[off+k-1], + * leaving elements b[off+k] through + * b[off+len-1] unaffected. + * + * In every case, elements b[0] through b[off] and elements b[off+len] + * through b[b.length-1] are unaffected. + * + * @param b A byte array which represents the buffer into which the data is read. + * @param off An int which represents the start offset in the byte array at which the data + * is written. + * @param len An int which represents the maximum number of bytes to read. + * @return An int which represents the total number of bytes read into the buffer, or -1 if there is no + * more data because the end of the stream has been reached. + * @throws IOException If the first byte cannot be read for any reason other than end of file, or if the input + * stream has been closed, or if some other I/O error occurs. + * @throws NullPointerException If the byte array b is null. + * @throws IndexOutOfBoundsException If off is negative, len is negative, or + * len is greater than + * b.length - off. + */ + @Override + public int read(final byte[] b, final int off, final int len) throws IOException { + AzurePath.ensureFileSystemOpen(path); + if (off < 0 || len < 0 || len > b.length - off) { + throw LOGGER.logExceptionAsError(new IndexOutOfBoundsException()); + } + try { + return this.blobInputStream.read(b, off, len); + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + /** + * Repositions this stream to the position at the time the mark method was last called on this input stream. Note + * repositioning the blob read stream will disable blob MD5 checking. + * + * @throws IOException If this stream has not been marked or if the mark has been invalidated. + */ + @Override + public synchronized void reset() throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + this.blobInputStream.reset(); + } catch (RuntimeException e) { + if (e.getMessage().equals("Stream mark expired.")) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + throw LoggingUtility.logError(LOGGER, e); + } + } + + /** + * Skips over and discards n bytes of data from this input stream. The skip method may, for a variety of reasons, + * end up skipping over some smaller number of bytes, possibly 0. This may result from any of a number of + * conditions; reaching end of file before n bytes have been skipped is only one possibility. The actual number of + * bytes skipped is returned. If n is negative, no bytes are skipped. + * + * Note repositioning the blob read stream will disable blob MD5 checking. + * + * @param n A long which represents the number of bytes to skip. + */ + @Override + public synchronized long skip(final long n) throws IOException { + AzurePath.ensureFileSystemOpen(path); + return this.blobInputStream.skip(n); + } + + BlobInputStream getBlobInputStream() { + return blobInputStream; + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/NioBlobOutputStream.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/NioBlobOutputStream.java new file mode 100644 index 00000000000..ae5c0fa02b1 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/NioBlobOutputStream.java @@ -0,0 +1,99 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +package com.azure.storage.blob.nio; + +import com.azure.core.util.logging.ClientLogger; +import com.azure.storage.blob.specialized.BlobOutputStream; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; + +/** + * Provides an OutputStream to write to a file stored as an Azure Blob. + */ +public final class NioBlobOutputStream extends OutputStream { + private static final ClientLogger LOGGER = new ClientLogger(NioBlobOutputStream.class); + + private final BlobOutputStream blobOutputStream; + private final Path path; + + NioBlobOutputStream(BlobOutputStream blobOutputStream, Path path) { + this.blobOutputStream = blobOutputStream; + this.path = path; + } + + @Override + public synchronized void write(int i) throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + this.blobOutputStream.write(i); + /* + BlobOutputStream only throws RuntimeException, and it doesn't preserve the cause, it only takes the message, + so we can't do any better than re-wrapping it in an IOException. + */ + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + @Override + public synchronized void write(byte[] b) throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + this.blobOutputStream.write(b); + /* + BlobOutputStream only throws RuntimeException, and it doesn't preserve the cause, it only takes the message, + so we can't do any better than re-wrapping it in an IOException. + */ + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + @Override + public synchronized void write(byte[] b, int off, int len) throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + this.blobOutputStream.write(b, off, len); + /* + BlobOutputStream only throws RuntimeException, and it doesn't preserve the cause, it only takes the message, + so we can't do any better than re-wrapping it in an IOException. + */ + } catch (RuntimeException e) { + if (e instanceof IndexOutOfBoundsException) { + throw LoggingUtility.logError(LOGGER, e); + } + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + @Override + public synchronized void flush() throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + this.blobOutputStream.flush(); + /* + BlobOutputStream only throws RuntimeException, and it doesn't preserve the cause, it only takes the message, + so we can't do any better than re-wrapping it in an IOException. + */ + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } + + @Override + public synchronized void close() throws IOException { + AzurePath.ensureFileSystemOpen(path); + try { + this.blobOutputStream.close(); + /* + BlobOutputStream only throws RuntimeException, and it doesn't preserve the cause, it only takes the message, + so we can't do any better than re-wrapping it in an IOException. + */ + } catch (RuntimeException e) { + throw LoggingUtility.logError(LOGGER, new IOException(e)); + } + } +} diff --git a/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/package-info.java b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/package-info.java new file mode 100644 index 00000000000..96cd1fbd627 --- /dev/null +++ b/azure-blob-nio/src/main/java/com/azure/storage/blob/nio/package-info.java @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +/** + * Package containing the classes for loading the AzureFileSystemProvider based on Azure Storage Blobs. + */ +package com.azure.storage.blob.nio; diff --git a/azure-blob-nio/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/azure-blob-nio/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider new file mode 100644 index 00000000000..5cc2b4ead14 --- /dev/null +++ b/azure-blob-nio/src/main/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -0,0 +1 @@ +com.azure.storage.blob.nio.AzureFileSystemProvider diff --git a/azure-blob-nio/src/main/resources/azure-storage-blob-nio.properties b/azure-blob-nio/src/main/resources/azure-storage-blob-nio.properties new file mode 100644 index 00000000000..ca812989b4f --- /dev/null +++ b/azure-blob-nio/src/main/resources/azure-storage-blob-nio.properties @@ -0,0 +1,2 @@ +name=${project.artifactId} +version=${project.version} diff --git a/azure-blob-nio/src/samples/java/com/azure/storage/blob/nio/ReadmeSamples.java b/azure-blob-nio/src/samples/java/com/azure/storage/blob/nio/ReadmeSamples.java new file mode 100644 index 00000000000..6c8c5e06e0b --- /dev/null +++ b/azure-blob-nio/src/samples/java/com/azure/storage/blob/nio/ReadmeSamples.java @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +package com.azure.storage.blob.nio; + +import com.azure.storage.blob.models.BlobHttpHeaders; +import com.azure.storage.common.StorageSharedKeyCredential; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.file.FileSystem; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; + +/** + * WARNING: MODIFYING THIS FILE WILL REQUIRE CORRESPONDING UPDATES TO README.md FILE. LINE NUMBERS + * ARE USED TO EXTRACT APPROPRIATE CODE SEGMENTS FROM THIS FILE. ADD NEW CODE AT THE BOTTOM TO AVOID CHANGING + * LINE NUMBERS OF EXISTING CODE SAMPLES. + * + * Code samples for the README.md + */ +public class ReadmeSamples { + + private static final String CONTAINER_STORES = "container1,container2"; // A comma separated list of container names + private static final StorageSharedKeyCredential SHARE_KEY_CREDENTIAL + = new StorageSharedKeyCredential("", ""); + private static final Map CONFIG = new HashMap() { + { + put(AzureFileSystem.AZURE_STORAGE_SHARED_KEY_CREDENTIAL, SHARE_KEY_CREDENTIAL); + put(AzureFileSystem.AZURE_STORAGE_FILE_STORES, CONTAINER_STORES); + } + }; + private FileSystem myFs = FileSystems.newFileSystem(new URI("azb://?endpoint= config = new HashMap<>(); + String stores = ","; // A comma separated list of container names + StorageSharedKeyCredential credential = new StorageSharedKeyCredential(" attributes = Files.readAttributes(filePath, "azureBlob:metadata,headers"); + // END: readme-sample-readAttributesOnAFileString + } + + public void writeAttributesToAFile() throws IOException { + // BEGIN: readme-sample-writeAttributesToAFile + AzureBlobFileAttributeView view = Files.getFileAttributeView(filePath, AzureBlobFileAttributeView.class); + view.setMetadata(Collections.emptyMap()); + // END: readme-sample-writeAttributesToAFile + } + + public void writeAttributesToAFileString() throws IOException { + // BEGIN: readme-sample-writeAttributesToAFileString + Files.setAttribute(filePath, "azureBlob:blobHttpHeaders", new BlobHttpHeaders()); + // END: readme-sample-writeAttributesToAFileString + } +} diff --git a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala index 54849df4250..549bf761e38 100644 --- a/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendCacheHitCopyingActor.scala @@ -6,13 +6,19 @@ import cromwell.core.simpleton.WomValueSimpleton import cromwell.services.CallCaching.CallCachingEntryId object BackendCacheHitCopyingActor { - final case class CopyOutputsCommand(womValueSimpletons: Seq[WomValueSimpleton], jobDetritusFiles: Map[String, String], cacheHit: CallCachingEntryId, returnCode: Option[Int]) + final case class CopyOutputsCommand(womValueSimpletons: Seq[WomValueSimpleton], + jobDetritusFiles: Map[String, String], + cacheHit: CallCachingEntryId, + returnCode: Option[Int] + ) final case class CopyingOutputsFailedResponse(jobKey: JobKey, cacheCopyAttempt: Int, failure: CacheCopyFailure) sealed trait CacheCopyFailure + /** A cache hit copy was attempted but failed. */ final case class CopyAttemptError(failure: Throwable) extends CacheCopyFailure + /** Copying was requested for a blacklisted cache hit, however the cache hit copying actor found the hit had already * been blacklisted so no novel copy attempt was made. */ final case class BlacklistSkip(failureCategory: MetricableCacheCopyErrorCategory) extends CacheCopyFailure diff --git a/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala b/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala index b1099b07155..8a9daa3cc39 100644 --- a/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala +++ b/backend/src/main/scala/cromwell/backend/BackendInitializationData.scala @@ -15,12 +15,11 @@ object BackendInitializationData { * @tparam A The type to cast the initialization data. * @return The initialization data as the type A. */ - def as[A <: BackendInitializationData](initializationDataOption: Option[BackendInitializationData]): A = { + def as[A <: BackendInitializationData](initializationDataOption: Option[BackendInitializationData]): A = initializationDataOption match { case Some(initializationData) => initializationData.asInstanceOf[A] case None => throw new RuntimeException("Initialization data was not found.") } - } } object AllBackendInitializationData { @@ -30,5 +29,6 @@ object AllBackendInitializationData { // Holds initialization data for all backends initialized for a workflow. case class AllBackendInitializationData(data: Map[String, Option[BackendInitializationData]]) { def get(backendName: String): Option[BackendInitializationData] = data.get(backendName).flatten - def getWorkflowRoots(): Set[Path] = data.values.collect({case Some(i: StandardInitializationData) => i.workflowPaths.workflowRoot}).toSet[Path] + def getWorkflowRoots(): Set[Path] = + data.values.collect { case Some(i: StandardInitializationData) => i.workflowPaths.workflowRoot }.toSet[Path] } diff --git a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala index a58131363d3..493221ec3ce 100644 --- a/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendJobExecutionActor.scala @@ -33,7 +33,8 @@ object BackendJobExecutionActor { jobDetritusFiles: Option[Map[String, Path]], executionEvents: Seq[ExecutionEvent], dockerImageUsed: Option[String], - resultGenerationMode: ResultGenerationMode) extends BackendJobExecutionResponse + resultGenerationMode: ResultGenerationMode + ) extends BackendJobExecutionResponse sealed trait ResultGenerationMode case object RunOnBackend extends ResultGenerationMode @@ -41,25 +42,30 @@ object BackendJobExecutionActor { case object FetchedFromJobStore extends ResultGenerationMode case class JobAbortedResponse(jobKey: BackendJobDescriptorKey) extends BackendJobExecutionResponse - - sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { def throwable: Throwable; def returnCode: Option[Int] } - case class JobFailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) extends BackendJobFailedResponse - case class JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, - throwable: Throwable, - returnCode: Option[Int]) extends BackendJobFailedResponse - - // Reconnection Exceptions - case class JobReconnectionNotSupportedException(jobKey: BackendJobDescriptorKey) extends Exception( - s"This backend does not support job reconnection. The status of the underlying job for ${jobKey.tag} cannot be known." - ) with CromwellFatalExceptionMarker - case class JobNotFoundException(jobKey: BackendJobDescriptorKey) extends Exception ( - s"No backend job for ${jobKey.tag} could be found. The status of the underlying job cannot be known." - ) with CromwellFatalExceptionMarker + sealed trait BackendJobFailedResponse extends BackendJobExecutionResponse { + def throwable: Throwable; def returnCode: Option[Int] + } + case class JobFailedNonRetryableResponse(jobKey: JobKey, throwable: Throwable, returnCode: Option[Int]) + extends BackendJobFailedResponse + case class JobFailedRetryableResponse(jobKey: BackendJobDescriptorKey, throwable: Throwable, returnCode: Option[Int]) + extends BackendJobFailedResponse - def buildJobExecutionActorName(workflowId: WorkflowId, jobKey: BackendJobDescriptorKey) = { + // Reconnection Exceptions + case class JobReconnectionNotSupportedException(jobKey: BackendJobDescriptorKey) + extends Exception( + s"This backend does not support job reconnection. The status of the underlying job for ${jobKey.tag} cannot be known." + ) + with CromwellFatalExceptionMarker + + case class JobNotFoundException(jobKey: BackendJobDescriptorKey) + extends Exception( + s"No backend job for ${jobKey.tag} could be found. The status of the underlying job cannot be known." + ) + with CromwellFatalExceptionMarker + + def buildJobExecutionActorName(workflowId: WorkflowId, jobKey: BackendJobDescriptorKey) = s"$workflowId-BackendJobExecutionActor-${jobKey.tag}" - } } /** @@ -92,7 +98,9 @@ trait BackendJobExecutionActor extends BackendJobLifecycleActor with ActorLoggin */ def recover: Future[BackendJobExecutionResponse] = { log.warning("{} backend currently doesn't support recovering jobs. Starting {} again.", - jobTag, jobDescriptor.key.call.fullyQualifiedName) + jobTag, + jobDescriptor.key.call.fullyQualifiedName + ) execute } @@ -100,28 +108,24 @@ trait BackendJobExecutionActor extends BackendJobLifecycleActor with ActorLoggin * Tries to reconnect to a previously started job. This method differs from recover by sending a ReconnectionFailure * if it can't reconnect to the job for whatever reason. It should NOT execute the job if reconnection is impossible. */ - def reconnect: Future[BackendJobExecutionResponse] = { + def reconnect: Future[BackendJobExecutionResponse] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) - } /** * Similar to reconnect, except that if the reconnection succeeds and the job is still running, * an abort attempt should be made. */ - def reconnectToAborting: Future[BackendJobExecutionResponse] = { + def reconnectToAborting: Future[BackendJobExecutionResponse] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) - } /** * Abort a running job. */ - def abort(): Unit = { - log.warning("{} backend currently doesn't support abort for {}.", - jobTag, jobDescriptor.key.call.fullyQualifiedName) - } + def abort(): Unit = + log.warning("{} backend currently doesn't support abort for {}.", jobTag, jobDescriptor.key.call.fullyQualifiedName) - def evaluateOutputs(wdlFunctions: IoFunctionSet, - postMapper: WomValue => Try[WomValue] = v => Success(v))(implicit ec: ExecutionContext): EvaluatedJobOutputs = { + def evaluateOutputs(wdlFunctions: IoFunctionSet, postMapper: WomValue => Try[WomValue] = v => Success(v))(implicit + ec: ExecutionContext + ): EvaluatedJobOutputs = Await.result(OutputEvaluator.evaluateOutputs(jobDescriptor, wdlFunctions, postMapper), Duration.Inf) - } } diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala index 72b0c24a800..450179c43e4 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActor.scala @@ -30,7 +30,7 @@ trait BackendLifecycleActor extends Actor { /** * The execution context for the actor */ - protected implicit def ec: ExecutionContext = context.dispatcher + implicit protected def ec: ExecutionContext = context.dispatcher /** * The configuration for the backend, in the context of the entire Cromwell configuration file. @@ -39,7 +39,8 @@ trait BackendLifecycleActor extends Actor { protected def performActionThenRespond(operation: => Future[BackendWorkflowLifecycleActorResponse], onFailure: Throwable => BackendWorkflowLifecycleActorResponse, - andThen: => Unit = ()) = { + andThen: => Unit = () + ) = { val respondTo: ActorRef = sender() operation onComplete { case Success(r) => @@ -54,9 +55,9 @@ trait BackendLifecycleActor extends Actor { trait BackendWorkflowLifecycleActor extends BackendLifecycleActor with WorkflowLogging { - //For Logging and boilerplate - override lazy final val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId - override lazy final val rootWorkflowIdForLogging = workflowDescriptor.rootWorkflowId + // For Logging and boilerplate + final override lazy val workflowIdForLogging = workflowDescriptor.possiblyNotRootWorkflowId + final override lazy val rootWorkflowIdForLogging = workflowDescriptor.rootWorkflowId /** * The workflow descriptor for the workflow in which this Backend is being used diff --git a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala index 0d38d87b01f..dc06ec94f09 100644 --- a/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/BackendLifecycleActorFactory.scala @@ -25,7 +25,8 @@ trait BackendLifecycleActorFactory { */ def name: String - def nameForCallCachingPurposes: String = configurationDescriptor.backendConfig.getOrElse("name-for-call-caching-purposes", name) + def nameForCallCachingPurposes: String = + configurationDescriptor.backendConfig.getOrElse("name-for-call-caching-purposes", name) /** * Config values for the backend, and a pointer to the global config. @@ -44,7 +45,8 @@ trait BackendLifecycleActorFactory { ioActor: ActorRef, calls: Set[CommandCallNode], serviceRegistryActor: ActorRef, - restarting: Boolean): Option[Props] = None + restarting: Boolean + ): Option[Props] = None /* ****************************** */ /* Job Execution */ @@ -54,7 +56,8 @@ trait BackendLifecycleActorFactory { initializationData: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActor: Option[ActorRef]): Props + backendSingletonActor: Option[ActorRef] + ): Props lazy val jobExecutionTokenType: JobTokenType = { val concurrentJobLimit = configurationDescriptor.backendConfig.as[Option[Int]]("concurrent-job-limit") @@ -67,7 +70,8 @@ trait BackendLifecycleActorFactory { } lazy val jobRestartCheckTokenType: JobTokenType = { - val concurrentRestartCheckLimit = configurationDescriptor.globalConfig.as[Option[Int]]("system.job-restart-check-rate-control.max-jobs") + val concurrentRestartCheckLimit = + configurationDescriptor.globalConfig.as[Option[Int]]("system.job-restart-check-rate-control.max-jobs") // if defined, use per-backend hog-factor, otherwise use system-level value val hogFactor = configurationDescriptor.backendConfig.as[Option[Int]]("hog-factor") match { case Some(backendHogFactorValue) => backendHogFactorValue @@ -85,13 +89,16 @@ trait BackendLifecycleActorFactory { calls: Set[CommandCallNode], jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, - initializationData: Option[BackendInitializationData]): Option[Props] = None + initializationData: Option[BackendInitializationData] + ): Option[Props] = None /* ****************************** */ /* Call Caching */ /* ****************************** */ - def fileHashingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = None + def fileHashingActorProps + : Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = + None /** * Providing this method to generate Props for a cache hit copying actor is optional. @@ -102,7 +109,9 @@ trait BackendLifecycleActorFactory { * * Simples! */ - def cacheHitCopyingActorProps: Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props] = None + def cacheHitCopyingActorProps: Option[ + (BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props + ] = None /* ****************************** */ /* Misc. */ @@ -114,19 +123,26 @@ trait BackendLifecycleActorFactory { jobKey: BackendJobDescriptorKey, initializationData: Option[BackendInitializationData], ioActor: ActorRef, - ec: ExecutionContext): IoFunctionSet = NoIoFunctionSet + ec: ExecutionContext + ): IoFunctionSet = NoIoFunctionSet def pathBuilders(initializationDataOption: Option[BackendInitializationData]): PathBuilders = List.empty - def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { + def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = new WorkflowPathsWithDocker(workflowDescriptor, backendConfig).executionRoot - } - def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, initializationData: Option[BackendInitializationData]): Path = { + def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = new WorkflowPathsWithDocker(workflowDescriptor, backendConfig).workflowRoot - } - def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]): Set[RuntimeAttributeDefinition] = Set.empty + def runtimeAttributeDefinitions( + initializationDataOption: Option[BackendInitializationData] + ): Set[RuntimeAttributeDefinition] = Set.empty /** * A set of KV store keys that this backend requests that the engine lookup before running each job. @@ -136,13 +152,21 @@ trait BackendLifecycleActorFactory { /** * A set of KV store keys that are requested and looked up on behalf of all backends before running each job. */ - def defaultKeyValueStoreKeys: Seq[String] = Seq(BackendLifecycleActorFactory.FailedRetryCountKey, BackendLifecycleActorFactory.MemoryMultiplierKey) + def defaultKeyValueStoreKeys: Seq[String] = + Seq(BackendLifecycleActorFactory.FailedRetryCountKey, BackendLifecycleActorFactory.MemoryMultiplierKey) /* * Returns credentials that can be used to authenticate to a docker registry server * in order to obtain a docker hash. */ - def dockerHashCredentials(workflowDescriptor: BackendWorkflowDescriptor, initializationDataOption: Option[BackendInitializationData]): List[Any] = List.empty + def dockerHashCredentials(workflowDescriptor: BackendWorkflowDescriptor, + initializationDataOption: Option[BackendInitializationData] + ): List[Any] = List.empty + + /** + * Allows Cromwell to self-identify which cloud it's running on for runtime attribute purposes + */ + def platform: Option[Platform] = None } object BackendLifecycleActorFactory { diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala index 2f77c2ebc2a..04abe76e372 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowFinalizationActor.scala @@ -27,8 +27,8 @@ object BackendWorkflowFinalizationActor { */ trait BackendWorkflowFinalizationActor extends BackendWorkflowLifecycleActor with ActorLogging { - def receive: Receive = LoggingReceive { - case Finalize => performActionThenRespond(afterAll() map { _ => FinalizationSuccess }, onFailure = FinalizationFailed) + def receive: Receive = LoggingReceive { case Finalize => + performActionThenRespond(afterAll() map { _ => FinalizationSuccess }, onFailure = FinalizationFailed) } /** diff --git a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala index bc3dd963f63..9f59a451478 100644 --- a/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/BackendWorkflowInitializationActor.scala @@ -34,7 +34,8 @@ object BackendWorkflowInitializationActor { // Responses sealed trait BackendWorkflowInitializationActorResponse extends BackendWorkflowLifecycleActorResponse sealed trait InitializationResponse extends BackendWorkflowInitializationActorResponse - case class InitializationSuccess(backendInitializationData: Option[BackendInitializationData]) extends InitializationResponse + case class InitializationSuccess(backendInitializationData: Option[BackendInitializationData]) + extends InitializationResponse case class InitializationFailed(reason: Throwable) extends Exception with InitializationResponse /** @@ -70,24 +71,25 @@ object BackendWorkflowInitializationActor { * - It would be nice to memoize as much of the work that gets done here as possible so it doesn't have to all be * repeated when the various `FooRuntimeAttributes` classes are created, in the spirit of #1076. */ - def validateRuntimeAttributes( - taskName: String, - defaultRuntimeAttributes: Map[String, WomValue], - runtimeAttributes: Map[String, WomExpression], - runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] - ): ValidatedNel[RuntimeAttributeValidationFailure, Unit] = { - - //This map append will overwrite default key/values with runtime settings upon key collisions - val lookups = defaultRuntimeAttributes.safeMapValues(_.asWomExpression) ++ runtimeAttributes - - runtimeAttributeValidators.toList.traverse{ - case (attributeName, validator) => - val runtimeAttributeValue: Option[WomExpression] = lookups.get(attributeName) - validator(runtimeAttributeValue).fold( - validNel(()), - Invalid(NonEmptyList.of(RuntimeAttributeValidationFailure(taskName, attributeName, runtimeAttributeValue))) - ) - }.map(_ => ()) + def validateRuntimeAttributes( + taskName: String, + defaultRuntimeAttributes: Map[String, WomValue], + runtimeAttributes: Map[String, WomExpression], + runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] + ): ValidatedNel[RuntimeAttributeValidationFailure, Unit] = { + + // This map append will overwrite default key/values with runtime settings upon key collisions + val lookups = defaultRuntimeAttributes.safeMapValues(_.asWomExpression) ++ runtimeAttributes + + runtimeAttributeValidators.toList + .traverse { case (attributeName, validator) => + val runtimeAttributeValue: Option[WomExpression] = lookups.get(attributeName) + validator(runtimeAttributeValue).fold( + validNel(()), + Invalid(NonEmptyList.of(RuntimeAttributeValidationFailure(taskName, attributeName, runtimeAttributeValue))) + ) + } + .map(_ => ()) } } @@ -110,7 +112,9 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w * declarations will fail evaluation and return `true` from this predicate, even if the type could be determined * to be wrong with consideration of task declarations or inputs. */ - protected def womTypePredicate(valueRequired: Boolean, predicate: WomType => Boolean)(womExpressionMaybe: Option[WomExpression]): Boolean = { + protected def womTypePredicate(valueRequired: Boolean, predicate: WomType => Boolean)( + womExpressionMaybe: Option[WomExpression] + ): Boolean = womExpressionMaybe match { case None => !valueRequired case Some(womExpression: WomExpression) => @@ -119,29 +123,31 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w case Invalid(_) => true // If we can't evaluate it, we'll let it pass for now... } } - } /** * This predicate is only appropriate for validation during workflow initialization. The logic does not differentiate * between evaluation failures due to missing call inputs or evaluation failures due to malformed expressions, and will * return `true` in both cases. */ - protected def continueOnReturnCodePredicate(valueRequired: Boolean)(womExpressionMaybe: Option[WomValue]): Boolean = { - ContinueOnReturnCodeValidation.default(configurationDescriptor.backendRuntimeAttributesConfig).validateOptionalWomValue(womExpressionMaybe) - } + protected def continueOnReturnCodePredicate(valueRequired: Boolean)(womExpressionMaybe: Option[WomValue]): Boolean = + ContinueOnReturnCodeValidation + .default(configurationDescriptor.backendRuntimeAttributesConfig) + .validateOptionalWomValue(womExpressionMaybe) protected def runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] // FIXME: If a workflow executes jobs using multiple backends, // each backend will try to write its own workflow root and override any previous one. // They should be structured differently or at least be prefixed by the backend name - protected def publishWorkflowRoot(workflowRoot: String): Unit = { - serviceRegistryActor ! PutMetadataAction(MetadataEvent(MetadataKey(workflowDescriptor.id, None, WorkflowMetadataKeys.WorkflowRoot), MetadataValue(workflowRoot))) - } + protected def publishWorkflowRoot(workflowRoot: String): Unit = + serviceRegistryActor ! PutMetadataAction( + MetadataEvent(MetadataKey(workflowDescriptor.id, None, WorkflowMetadataKeys.WorkflowRoot), + MetadataValue(workflowRoot) + ) + ) protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] - def receive: Receive = LoggingReceive { case Initialize => performActionThenRespond(initSequence(), onFailure = InitializationFailed) case Abort => abortInitialization() @@ -154,10 +160,11 @@ trait BackendWorkflowInitializationActor extends BackendWorkflowLifecycleActor w for { defaultRuntimeAttributes <- coerceDefaultRuntimeAttributes(workflowDescriptor.workflowOptions) |> Future.fromTry _ taskList = calls.toList.map(_.callable).map(t => t.name -> t.runtimeAttributes.attributes) - _ <- taskList. - traverse{ - case (name, runtimeAttributes) => validateRuntimeAttributes(name, defaultRuntimeAttributes, runtimeAttributes, runtimeAttributeValidators) - }.toFuture(errors => RuntimeAttributeValidationFailures(errors.toList)) + _ <- taskList + .traverse { case (name, runtimeAttributes) => + validateRuntimeAttributes(name, defaultRuntimeAttributes, runtimeAttributes, runtimeAttributeValidators) + } + .toFuture(errors => RuntimeAttributeValidationFailures(errors.toList)) _ <- validate() data <- beforeAll() } yield InitializationSuccess(data) diff --git a/backend/src/main/scala/cromwell/backend/Command.scala b/backend/src/main/scala/cromwell/backend/Command.scala index 40310c68485..c523500b7ca 100644 --- a/backend/src/main/scala/cromwell/backend/Command.scala +++ b/backend/src/main/scala/cromwell/backend/Command.scala @@ -3,7 +3,6 @@ package cromwell.backend import common.validation.ErrorOr._ import common.validation.Validation._ import wom.InstantiatedCommand -import wom.callable.RuntimeEnvironment import wom.expression.IoFunctionSet import wom.values.{WomEvaluatedCallInputs, WomValue} @@ -24,11 +23,11 @@ object Command { */ def instantiate(jobDescriptor: BackendJobDescriptor, callEngineFunction: IoFunctionSet, - inputsPreProcessor: WomEvaluatedCallInputs => Try[WomEvaluatedCallInputs] = (i: WomEvaluatedCallInputs) => Success(i), - valueMapper: WomValue => WomValue, - runtimeEnvironment: RuntimeEnvironment): ErrorOr[InstantiatedCommand] = { + inputsPreProcessor: WomEvaluatedCallInputs => Try[WomEvaluatedCallInputs] = + (i: WomEvaluatedCallInputs) => Success(i), + valueMapper: WomValue => WomValue + ): ErrorOr[InstantiatedCommand] = inputsPreProcessor(jobDescriptor.evaluatedTaskInputs).toErrorOr flatMap { mappedInputs => - jobDescriptor.taskCall.callable.instantiateCommand(mappedInputs, callEngineFunction, valueMapper, runtimeEnvironment) + jobDescriptor.taskCall.callable.instantiateCommand(mappedInputs, callEngineFunction, valueMapper) } - } } diff --git a/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala b/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala index 2580d9218fc..79ccc698b05 100644 --- a/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala +++ b/backend/src/main/scala/cromwell/backend/FileSizeTooBig.scala @@ -1,4 +1,3 @@ package cromwell.backend case class FileSizeTooBig(override val getMessage: String) extends Exception - diff --git a/backend/src/main/scala/cromwell/backend/MinimumRuntimeSettings.scala b/backend/src/main/scala/cromwell/backend/MinimumRuntimeSettings.scala new file mode 100644 index 00000000000..67c18324ef8 --- /dev/null +++ b/backend/src/main/scala/cromwell/backend/MinimumRuntimeSettings.scala @@ -0,0 +1,13 @@ +package cromwell.backend + +import eu.timepit.refined.api.Refined +import eu.timepit.refined.numeric.Positive +import eu.timepit.refined.refineMV +import wdl4s.parser.MemoryUnit +import wom.format.MemorySize + +case class MinimumRuntimeSettings(cores: Int Refined Positive = refineMV(1), + ram: MemorySize = MemorySize(4, MemoryUnit.GB), + outputPathSize: Long = Long.MaxValue, + tempPathSize: Long = Long.MaxValue +) diff --git a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala b/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala index 2bf215fb0fa..8c799e36ad5 100644 --- a/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala +++ b/backend/src/main/scala/cromwell/backend/OutputEvaluator.scala @@ -26,15 +26,20 @@ object OutputEvaluator { def evaluateOutputs(jobDescriptor: BackendJobDescriptor, ioFunctions: IoFunctionSet, - postMapper: WomValue => Try[WomValue] = v => Success(v))(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = { + postMapper: WomValue => Try[WomValue] = v => Success(v) + )(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = { val taskInputValues: Map[String, WomValue] = jobDescriptor.localInputs - def foldFunction(accumulatedOutputs: Try[ErrorOr[List[(OutputPort, WomValue)]]], output: ExpressionBasedOutputPort) = accumulatedOutputs flatMap { accumulated => + def foldFunction(accumulatedOutputs: Try[ErrorOr[List[(OutputPort, WomValue)]]], + output: ExpressionBasedOutputPort + ) = accumulatedOutputs flatMap { accumulated => // Extract the valid pairs from the job outputs accumulated so far, and add to it the inputs (outputs can also reference inputs) val allKnownValues: Map[String, WomValue] = accumulated match { case Valid(outputs) => // The evaluateValue methods needs a Map[String, WomValue], use the output port name for already computed outputs - outputs.toMap[OutputPort, WomValue].map({ case (port, value) => port.internalName -> value }) ++ taskInputValues + outputs.toMap[OutputPort, WomValue].map { case (port, value) => + port.internalName -> value + } ++ taskInputValues case Invalid(_) => taskInputValues } @@ -45,22 +50,24 @@ object OutputEvaluator { } // Attempt to coerce the womValue to the desired output type - def coerceOutputValue(womValue: WomValue, coerceTo: WomType): OutputResult[WomValue] = { + def coerceOutputValue(womValue: WomValue, coerceTo: WomType): OutputResult[WomValue] = fromEither[Try]( // TODO WOM: coerceRawValue should return an ErrorOr - coerceTo.coerceRawValue(womValue).toEither.leftMap(t => NonEmptyList.one(t.getClass.getSimpleName + ": " + t.getMessage)) + coerceTo + .coerceRawValue(womValue) + .toEither + .leftMap(t => NonEmptyList.one(t.getClass.getSimpleName + ": " + t.getMessage)) ) - } /* - * Go through evaluation, coercion and post processing. - * Transform the result to a validated Try[ErrorOr[(String, WomValue)]] with toValidated - * If we have a valid pair, add it to the previously accumulated outputs, otherwise combine the Nels of errors + * Go through evaluation, coercion and post processing. + * Transform the result to a validated Try[ErrorOr[(String, WomValue)]] with toValidated + * If we have a valid pair, add it to the previously accumulated outputs, otherwise combine the Nels of errors */ val evaluated = for { evaluated <- evaluateOutputExpression coerced <- coerceOutputValue(evaluated, output.womType) - postProcessed <- EitherT { postMapper(coerced).map(_.validNelCheck) }: OutputResult[WomValue] + postProcessed <- EitherT(postMapper(coerced).map(_.validNelCheck)): OutputResult[WomValue] pair = output -> postProcessed } yield pair @@ -73,24 +80,27 @@ object OutputEvaluator { val emptyValue = Success(List.empty[(OutputPort, WomValue)].validNel): Try[ErrorOr[List[(OutputPort, WomValue)]]] // Fold over the outputs to evaluate them in order, map the result to an EvaluatedJobOutputs - def fromOutputPorts: EvaluatedJobOutputs = jobDescriptor.taskCall.expressionBasedOutputPorts.foldLeft(emptyValue)(foldFunction) match { - case Success(Valid(outputs)) => ValidJobOutputs(CallOutputs(outputs.toMap)) - case Success(Invalid(errors)) => InvalidJobOutputs(errors) - case Failure(exception) => JobOutputsEvaluationException(exception) - } + def fromOutputPorts: EvaluatedJobOutputs = + jobDescriptor.taskCall.expressionBasedOutputPorts.foldLeft(emptyValue)(foldFunction) match { + case Success(Valid(outputs)) => ValidJobOutputs(CallOutputs(outputs.toMap)) + case Success(Invalid(errors)) => InvalidJobOutputs(errors) + case Failure(exception) => JobOutputsEvaluationException(exception) + } /* - * Because Cromwell doesn't trust anyone, if custom evaluation is provided, - * still make sure that all the output ports have been filled with values + * Because Cromwell doesn't trust anyone, if custom evaluation is provided, + * still make sure that all the output ports have been filled with values */ def validateCustomEvaluation(outputs: Map[OutputPort, WomValue]): EvaluatedJobOutputs = { - def toError(outputPort: OutputPort) = s"Missing output value for ${outputPort.identifier.fullyQualifiedName.value}" + def toError(outputPort: OutputPort) = + s"Missing output value for ${outputPort.identifier.fullyQualifiedName.value}" jobDescriptor.taskCall.expressionBasedOutputPorts.diff(outputs.keySet.toList) match { case Nil => val errorMessagePrefix = "Error applying postMapper in short-circuit output evaluation" - TryUtil.sequenceMap(outputs map { case (k, v) => (k, postMapper(v))}, errorMessagePrefix) match { - case Failure(e) => InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).toIndexedSeq.map(_.toString):_*)) + TryUtil.sequenceMap(outputs map { case (k, v) => (k, postMapper(v)) }, errorMessagePrefix) match { + case Failure(e) => + InvalidJobOutputs(NonEmptyList.of(e.getMessage, e.getStackTrace.take(5).toIndexedSeq.map(_.toString): _*)) case Success(postMappedOutputs) => ValidJobOutputs(CallOutputs(postMappedOutputs)) } case head :: tail => InvalidJobOutputs(NonEmptyList.of(toError(head), tail.map(toError): _*)) @@ -98,18 +108,20 @@ object OutputEvaluator { } /* - * See if the task definition has "short-circuit" for the default output evaluation. - * In the case of CWL for example, this gives a chance to look for cwl.output.json and use it as the output of the tool, - * instead of the default behavior of going over each output port of the task and evaluates their expression. - * If the "customOutputEvaluation" returns None (which will happen if the cwl.output.json is not there, as well as for all WDL workflows), - * we fallback to the default behavior. + * See if the task definition has "short-circuit" for the default output evaluation. + * In the case of CWL for example, this gives a chance to look for cwl.output.json and use it as the output of the tool, + * instead of the default behavior of going over each output port of the task and evaluates their expression. + * If the "customOutputEvaluation" returns None (which will happen if the cwl.output.json is not there, as well as for all WDL workflows), + * we fallback to the default behavior. */ - jobDescriptor.taskCall.customOutputEvaluation(taskInputValues, ioFunctions, ec).value - .map({ + jobDescriptor.taskCall + .customOutputEvaluation(taskInputValues, ioFunctions, ec) + .value + .map { case Some(Right(outputs)) => validateCustomEvaluation(outputs) case Some(Left(errors)) => InvalidJobOutputs(errors) // If it returns an empty value, fallback to canonical output evaluation case None => fromOutputPorts - }) + } } } diff --git a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala index 33e6e2db020..37bfa3aa7b8 100644 --- a/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala +++ b/backend/src/main/scala/cromwell/backend/RuntimeAttributeDefinition.scala @@ -6,6 +6,7 @@ import cromwell.util.JsonFormatting.WomValueJsonFormatter import common.validation.ErrorOr.ErrorOr import wom.callable.Callable.InputDefinition import wom.expression.IoFunctionSet +import wom.values.WomObject import wom.values.WomValue import wom.{RuntimeAttributes, WomExpressionException} @@ -20,30 +21,80 @@ case class RuntimeAttributeDefinition(name: String, factoryDefault: Option[WomVa object RuntimeAttributeDefinition { + /** + * "Evaluate" means hydrating the runtime attributes with information from the inputs + * @param unevaluated WOM expressions that may or may not reference inputs + * @param wdlFunctions The set of IO for the current backend + * @param evaluatedInputs The inputs + * @param platform Optional, directs platform-based prioritization + * @return Evaluated + */ def evaluateRuntimeAttributes(unevaluated: RuntimeAttributes, wdlFunctions: IoFunctionSet, - evaluatedInputs: Map[InputDefinition, WomValue]): ErrorOr[Map[String, WomValue]] = { + evaluatedInputs: Map[InputDefinition, WomValue], + platform: Option[Platform] = None + ): ErrorOr[Map[String, WomValue]] = { import common.validation.ErrorOr._ val inputsMap = evaluatedInputs map { case (x, y) => x.name -> y } - unevaluated.attributes.traverseValues(_.evaluateValue(inputsMap, wdlFunctions)) + val evaluated = unevaluated.attributes.traverseValues(_.evaluateValue(inputsMap, wdlFunctions)) + + // Platform mapping must come after evaluation because we need to evaluate + // e.g. `gcp: userDefinedObject` to find out what its runtime value is. + // The type system informs us of this because a `WomExpression` in `unevaluated` + // cannot be safely read as a `WomObject` with a `values` map until evaluation + evaluated.map(e => applyPlatform(e, platform)) + } + + def applyPlatform(attributes: Map[String, WomValue], maybePlatform: Option[Platform]): Map[String, WomValue] = { + + def extractPlatformAttributes(platform: Platform): Map[String, WomValue] = + attributes.get(platform.runtimeKey) match { + case Some(obj: WomObject) => + // WDL spec: "Use objects to avoid collisions" + // https://github.com/openwdl/wdl/blob/wdl-1.1/SPEC.md#conventions-and-best-practices + obj.values + case _ => + // A malformed non-object override such as gcp: "banana" is ignored + Map.empty + } + + val platformAttributes = maybePlatform match { + case Some(platform) => + extractPlatformAttributes(platform) + case None => + Map.empty + } + + // We've scooped our desired platform, now delete "azure", "gcp", etc. + val originalAttributesWithoutPlatforms: Map[String, WomValue] = + attributes -- Platform.all.map(_.runtimeKey) + + // With `++` keys from the RHS overwrite duplicates in LHS, which is what we want + // RHS `Map.empty` is a no-op + originalAttributesWithoutPlatforms ++ platformAttributes } def buildMapBasedLookup(evaluatedDeclarations: Map[InputDefinition, Try[WomValue]])(identifier: String): WomValue = { val successfulEvaluations = evaluatedDeclarations collect { case (k, v) if v.isSuccess => k.name -> v.get } - successfulEvaluations.getOrElse(identifier, throw new WomExpressionException(s"Could not resolve variable $identifier as a task input")) + successfulEvaluations.getOrElse( + identifier, + throw new WomExpressionException(s"Could not resolve variable $identifier as a task input") + ) } - def addDefaultsToAttributes(runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition], workflowOptions: WorkflowOptions) - (specifiedAttributes: Map[LocallyQualifiedName, WomValue]): Map[LocallyQualifiedName, WomValue] = { + def addDefaultsToAttributes(runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition], + workflowOptions: WorkflowOptions + )(specifiedAttributes: Map[LocallyQualifiedName, WomValue]): Map[LocallyQualifiedName, WomValue] = { import WomValueJsonFormatter._ def isUnspecifiedAttribute(name: String) = !specifiedAttributes.contains(name) val missing = runtimeAttributeDefinitions filter { x => isUnspecifiedAttribute(x.name) } val defaults = missing map { x => (x, workflowOptions.getDefaultRuntimeOption(x.name)) } collect { - case (runtimeAttributeDefinition, Success(jsValue)) => runtimeAttributeDefinition.name -> jsValue.convertTo[WomValue] + case (runtimeAttributeDefinition, Success(jsValue)) => + runtimeAttributeDefinition.name -> jsValue.convertTo[WomValue] case (RuntimeAttributeDefinition(name, Some(factoryDefault), _), _) => name -> factoryDefault } specifiedAttributes ++ defaults diff --git a/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala b/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala deleted file mode 100644 index c6c4cd22c21..00000000000 --- a/backend/src/main/scala/cromwell/backend/RuntimeEnvironment.scala +++ /dev/null @@ -1,59 +0,0 @@ -package cromwell.backend - -import java.util.UUID - -import cromwell.backend.io.JobPaths -import cromwell.backend.validation.{CpuValidation, MemoryValidation} -import cromwell.core.path.Path -import eu.timepit.refined.api.Refined -import eu.timepit.refined.numeric.Positive -import eu.timepit.refined.refineMV -import wdl4s.parser.MemoryUnit -import wom.callable.RuntimeEnvironment -import wom.format.MemorySize -import wom.values.WomValue - -object RuntimeEnvironmentBuilder { - - def apply(runtimeAttributes: Map[String, WomValue], callRoot: Path, callExecutionRoot: Path): MinimumRuntimeSettings => RuntimeEnvironment = { - minimums => - - val outputPath: String = callExecutionRoot.pathAsString - - val tempPath: String = { - val uuid = UUID.randomUUID().toString - val hash = uuid.substring(0, uuid.indexOf('-')) - callRoot.resolve(s"tmp.$hash").pathAsString - } - - val cores: Int Refined Positive = CpuValidation.instanceMin.validate(runtimeAttributes).getOrElse(minimums.cores) - - val memoryInMB: Double = - MemoryValidation.instance(). - validate(runtimeAttributes). - map(_.to(MemoryUnit.MB).amount). - getOrElse(minimums.ram.amount) - - //TODO: Read these from somewhere else - val outputPathSize: Long = minimums.outputPathSize - - val tempPathSize: Long = minimums.outputPathSize - - RuntimeEnvironment(outputPath, tempPath, cores, memoryInMB, outputPathSize, tempPathSize) - } - - /** - * Per the spec: - * - * "For cores, ram, outdirSize and tmpdirSize, if an implementation can't provide the actual number of reserved cores - * during the expression evaluation time, it should report back the minimal requested amount." - */ - def apply(runtimeAttributes: Map[String, WomValue], jobPaths: JobPaths): MinimumRuntimeSettings => RuntimeEnvironment = { - this.apply(runtimeAttributes, jobPaths.callRoot, jobPaths.callExecutionRoot) - } -} - -case class MinimumRuntimeSettings(cores: Int Refined Positive = refineMV(1), - ram: MemorySize = MemorySize(4, MemoryUnit.GB), - outputPathSize: Long = Long.MaxValue, - tempPathSize: Long = Long.MaxValue) diff --git a/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala b/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala index 074895e366a..22db6d7f92a 100644 --- a/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala +++ b/backend/src/main/scala/cromwell/backend/SlowJobWarning.scala @@ -12,20 +12,21 @@ trait SlowJobWarning { this: Actor with ActorLogging => def slowJobWarningReceive: Actor.Receive = { case WarnAboutSlownessAfter(jobId, duration) => alreadyWarned = false - warningDetails = Option(WarningDetails(jobId, OffsetDateTime.now(), OffsetDateTime.now().plusSeconds(duration.toSeconds))) + warningDetails = Option( + WarningDetails(jobId, OffsetDateTime.now(), OffsetDateTime.now().plusSeconds(duration.toSeconds)) + ) case WarnAboutSlownessIfNecessary => handleWarnMessage() } var warningDetails: Option[WarningDetails] = None var alreadyWarned: Boolean = false - def warnAboutSlowJobIfNecessary(jobId: String) = { + def warnAboutSlowJobIfNecessary(jobId: String) = // Don't do anything here because we might need to update state. // Instead, send a message and handle this in the receive block. self ! WarnAboutSlownessIfNecessary - } - private def handleWarnMessage(): Unit = { + private def handleWarnMessage(): Unit = if (!alreadyWarned) { warningDetails match { case Some(WarningDetails(jobId, startTime, warningTime)) if OffsetDateTime.now().isAfter(warningTime) => @@ -34,7 +35,6 @@ trait SlowJobWarning { this: Actor with ActorLogging => case _ => // Nothing to do } } - } } diff --git a/backend/src/main/scala/cromwell/backend/WriteFunctions.scala b/backend/src/main/scala/cromwell/backend/WriteFunctions.scala index 572a417c047..5a8224d37a6 100644 --- a/backend/src/main/scala/cromwell/backend/WriteFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/WriteFunctions.scala @@ -24,13 +24,14 @@ trait WriteFunctions extends PathFactory with IoFunctionSet with AsyncIoFunction */ def writeDirectory: Path - private lazy val _writeDirectory = if (isDocker) writeDirectory.createPermissionedDirectories() else writeDirectory.createDirectories() + private lazy val _writeDirectory = + if (isDocker) writeDirectory.createPermissionedDirectories() else writeDirectory.createDirectories() override def createTemporaryDirectory(name: Option[String]) = { val tempDirPath = _writeDirectory / name.getOrElse(UUID.randomUUID().toString) // This is evil, but has the added advantage to work both for cloud and local val tempDirHiddenFile = tempDirPath / ".file" - asyncIo.writeAsync(tempDirHiddenFile, "", OpenOptions.default) as { tempDirPath.pathAsString } + asyncIo.writeAsync(tempDirHiddenFile, "", OpenOptions.default) as tempDirPath.pathAsString } protected def writeAsync(file: Path, content: String) = asyncIo.writeAsync(file, content, OpenOptions.default) @@ -38,21 +39,22 @@ trait WriteFunctions extends PathFactory with IoFunctionSet with AsyncIoFunction override def writeFile(path: String, content: String): Future[WomSingleFile] = { val file = _writeDirectory / path asyncIo.existsAsync(file) flatMap { - case false => writeAsync(file, content) as { WomSingleFile(file.pathAsString) } + case false => writeAsync(file, content) as WomSingleFile(file.pathAsString) case true => Future.successful(WomSingleFile(file.pathAsString)) } } private val relativeToLocal = System.getProperty("user.dir").ensureSlashed - def relativeToAbsolutePath(pathFrom: String): String = if (buildPath(pathFrom).isAbsolute) pathFrom else relativeToLocal + pathFrom + def relativeToAbsolutePath(pathFrom: String): String = + if (buildPath(pathFrom).isAbsolute) pathFrom else relativeToLocal + pathFrom override def copyFile(pathFrom: String, targetName: String): Future[WomSingleFile] = { val source = buildPath(relativeToAbsolutePath(pathFrom)) val destination = _writeDirectory / targetName - asyncIo.copyAsync(source, destination).as(WomSingleFile(destination.pathAsString)) recoverWith { - case e => Future.failed(new Exception(s"Could not copy ${source.toAbsolutePath} to ${destination.toAbsolutePath}", e)) + asyncIo.copyAsync(source, destination).as(WomSingleFile(destination.pathAsString)) recoverWith { case e => + Future.failed(new Exception(s"Could not copy ${source.toAbsolutePath} to ${destination.toAbsolutePath}", e)) } } } diff --git a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala index 241081f5a02..2872633d9bd 100644 --- a/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/async/AsyncBackendJobExecutionActor.scala @@ -1,6 +1,5 @@ package cromwell.backend.async - import java.util.concurrent.ExecutionException import akka.actor.{Actor, ActorLogging, ActorRef} @@ -18,10 +17,12 @@ import scala.util.{Failure, Success} object AsyncBackendJobExecutionActor { sealed trait AsyncBackendJobExecutionActorMessage - private final case class IssuePollRequest(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage - private final case class PollResponseReceived(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage - private final case class FailAndStop(reason: Throwable) extends AsyncBackendJobExecutionActorMessage - private final case class Finish(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage + final private case class IssuePollRequest(executionHandle: ExecutionHandle) + extends AsyncBackendJobExecutionActorMessage + final private case class PollResponseReceived(executionHandle: ExecutionHandle) + extends AsyncBackendJobExecutionActorMessage + final private case class FailAndStop(reason: Throwable) extends AsyncBackendJobExecutionActorMessage + final private case class Finish(executionHandle: ExecutionHandle) extends AsyncBackendJobExecutionActorMessage trait JobId @@ -57,27 +58,24 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging with SlowJob def isTransient(throwable: Throwable): Boolean = false - private def withRetry[A](work: () => Future[A], backOff: SimpleExponentialBackoff): Future[A] = { + private def withRetry[A](work: () => Future[A], backOff: SimpleExponentialBackoff): Future[A] = Retry.withRetry(work, isTransient = isTransient, isFatal = isFatal, backoff = backOff)(context.system) - } - private def robustExecuteOrRecover(mode: ExecutionMode) = { + private def robustExecuteOrRecover(mode: ExecutionMode) = withRetry(() => executeOrRecover(mode), executeOrRecoverBackOff) onComplete { case Success(h) => self ! IssuePollRequest(h) case Failure(t) => self ! FailAndStop(t) } - } def pollBackOff: SimpleExponentialBackoff def executeOrRecoverBackOff: SimpleExponentialBackoff - private def robustPoll(handle: ExecutionHandle) = { + private def robustPoll(handle: ExecutionHandle) = withRetry(() => poll(handle), pollBackOff) onComplete { case Success(h) => self ! PollResponseReceived(h) case Failure(t) => self ! FailAndStop(t) } - } private def failAndStop(t: Throwable) = { completionPromise.success(JobFailedNonRetryableResponse(jobDescriptor.key, t, None)) @@ -94,7 +92,16 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging with SlowJob context.system.scheduler.scheduleOnce(pollBackOff.backoffMillis.millis, self, IssuePollRequest(handle)) () case Finish(SuccessfulExecutionHandle(outputs, returnCode, jobDetritusFiles, executionEvents, _)) => - completionPromise.success(JobSucceededResponse(jobDescriptor.key, Some(returnCode), outputs, Option(jobDetritusFiles), executionEvents, dockerImageUsed, resultGenerationMode = RunOnBackend)) + completionPromise.success( + JobSucceededResponse(jobDescriptor.key, + Some(returnCode), + outputs, + Option(jobDetritusFiles), + executionEvents, + dockerImageUsed, + resultGenerationMode = RunOnBackend + ) + ) context.stop(self) case Finish(FailedNonRetryableExecutionHandle(throwable, returnCode, _)) => completionPromise.success(JobFailedNonRetryableResponse(jobDescriptor.key, throwable, returnCode)) @@ -133,5 +140,5 @@ trait AsyncBackendJobExecutionActor { this: Actor with ActorLogging with SlowJob def jobDescriptor: BackendJobDescriptor - protected implicit def ec: ExecutionContext + implicit protected def ec: ExecutionContext } diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala index ab1354e4323..ff6a088b1f9 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionHandle.scala @@ -15,8 +15,7 @@ sealed trait ExecutionHandle { def result: ExecutionResult } -final case class PendingExecutionHandle[BackendJobId <: JobId, BackendRunInfo, BackendRunState] -( +final case class PendingExecutionHandle[BackendJobId <: JobId, BackendRunInfo, BackendRunState]( jobDescriptor: BackendJobDescriptor, pendingJob: BackendJobId, runInfo: Option[BackendRunInfo], @@ -30,7 +29,8 @@ final case class SuccessfulExecutionHandle(outputs: CallOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], - resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionHandle { + resultsClonedFrom: Option[BackendJobDescriptor] = None +) extends ExecutionHandle { override val isDone = true override val result = SuccessfulExecution(outputs, returnCode, jobDetritusFiles, executionEvents, resultsClonedFrom) } @@ -41,7 +41,8 @@ sealed trait FailedExecutionHandle extends ExecutionHandle { final case class FailedNonRetryableExecutionHandle(throwable: Throwable, returnCode: Option[Int] = None, - override val kvPairsToSave: Option[Seq[KvPair]]) extends FailedExecutionHandle { + override val kvPairsToSave: Option[Seq[KvPair]] +) extends FailedExecutionHandle { override val isDone = true override val result = NonRetryableExecution(throwable, returnCode) @@ -49,7 +50,8 @@ final case class FailedNonRetryableExecutionHandle(throwable: Throwable, final case class FailedRetryableExecutionHandle(throwable: Throwable, returnCode: Option[Int] = None, - override val kvPairsToSave: Option[Seq[KvPair]]) extends FailedExecutionHandle { + override val kvPairsToSave: Option[Seq[KvPair]] +) extends FailedExecutionHandle { override val isDone = true override val result = RetryableExecution(throwable, returnCode) diff --git a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala index fc8f95e049b..b3c4bcda692 100644 --- a/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala +++ b/backend/src/main/scala/cromwell/backend/async/ExecutionResult.scala @@ -16,7 +16,8 @@ final case class SuccessfulExecution(outputs: CallOutputs, returnCode: Int, jobDetritusFiles: Map[String, Path], executionEvents: Seq[ExecutionEvent], - resultsClonedFrom: Option[BackendJobDescriptor] = None) extends ExecutionResult + resultsClonedFrom: Option[BackendJobDescriptor] = None +) extends ExecutionResult /** * A user-requested abort of the command. diff --git a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala index 8dd9aef3544..a40f127ce2e 100644 --- a/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala +++ b/backend/src/main/scala/cromwell/backend/async/KnownJobFailureException.scala @@ -9,8 +9,14 @@ abstract class KnownJobFailureException extends Exception { def stderrPath: Option[Path] } -final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Option[Path]) extends KnownJobFailureException { - override def getMessage = s"Job $jobTag exited with return code $returnCode which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details." +final case class WrongReturnCode(jobTag: String, returnCode: Int, stderrPath: Option[Path]) + extends KnownJobFailureException { + override def getMessage = + s"Job $jobTag exited with return code $returnCode which has not been declared as a valid return code. See 'continueOnReturnCode' runtime attribute for more details." +} + +final case class UnExpectedStatus(jobTag: String, returnCode: Int, jobStatus: String, stderrPath: Option[Path]) extends KnownJobFailureException { + override def getMessage = s"Job $jobTag exited with success code '$returnCode' but failed status '$jobStatus'. Suspecting spot kill and retrying." } final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderrPath: Option[Path]) extends KnownJobFailureException { @@ -22,48 +28,62 @@ final case class ReturnCodeIsNotAnInt(jobTag: String, returnCode: String, stderr } } -final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Option[Path]) extends KnownJobFailureException { - override def getMessage = s"stderr for job $jobTag has length $stderrLength and 'failOnStderr' runtime attribute was true." +final case class StderrNonEmpty(jobTag: String, stderrLength: Long, stderrPath: Option[Path]) + extends KnownJobFailureException { + override def getMessage = + s"stderr for job $jobTag has length $stderrLength and 'failOnStderr' runtime attribute was true." } final case class RetryWithMoreMemory(jobTag: String, stderrPath: Option[Path], memoryRetryErrorKeys: Option[List[String]], - logger: LoggingAdapter) extends KnownJobFailureException { + logger: LoggingAdapter +) extends KnownJobFailureException { val errorKeysAsString = memoryRetryErrorKeys match { case None => // this should not occur at this point as one would reach this error class only if Cromwell found one of the // `memory-retry-error-keys` in `stderr` of the task, which is only checked if the `memory-retry-error-keys` // are instantiated in Cromwell config - logger.error(s"Programmer error: found one of the `system.memory-retry-error-keys` in the `stderr` of task but " + - s"didn't find the error keys while generating the exception!") + logger.error( + s"Programmer error: found one of the `system.memory-retry-error-keys` in the `stderr` of task but " + + s"didn't find the error keys while generating the exception!" + ) "" case Some(keys) => keys.mkString(": [", ",", "]") } - override def getMessage = s"stderr for job `$jobTag` contained one of the `memory-retry-error-keys${errorKeysAsString}` specified in " + - s"the Cromwell config. Job might have run out of memory." + override def getMessage = + s"stderr for job `$jobTag` contained one of the `memory-retry-error-keys${errorKeysAsString}` specified in " + + s"the Cromwell config. Job might have run out of memory." } - object RuntimeAttributeValidationFailure { def apply(jobTag: String, runtimeAttributeName: String, - runtimeAttributeValue: Option[WomExpression]): RuntimeAttributeValidationFailure = RuntimeAttributeValidationFailure(jobTag, runtimeAttributeName, runtimeAttributeValue, None) + runtimeAttributeValue: Option[WomExpression] + ): RuntimeAttributeValidationFailure = + RuntimeAttributeValidationFailure(jobTag, runtimeAttributeName, runtimeAttributeValue, None) } final case class RuntimeAttributeValidationFailure private (jobTag: String, runtimeAttributeName: String, runtimeAttributeValue: Option[WomExpression], - stderrPath: Option[Path]) extends KnownJobFailureException { - override def getMessage = s"Task $jobTag has an invalid runtime attribute $runtimeAttributeName = ${runtimeAttributeValue map { _.evaluateValue(Map.empty, NoIoFunctionSet)} getOrElse "!! NOT FOUND !!"}" + stderrPath: Option[Path] +) extends KnownJobFailureException { + override def getMessage = + s"Task $jobTag has an invalid runtime attribute $runtimeAttributeName = ${runtimeAttributeValue map { + _.evaluateValue(Map.empty, NoIoFunctionSet) + } getOrElse "!! NOT FOUND !!"}" } -final case class RuntimeAttributeValidationFailures(throwables: List[RuntimeAttributeValidationFailure]) extends KnownJobFailureException with ThrowableAggregation { +final case class RuntimeAttributeValidationFailures(throwables: List[RuntimeAttributeValidationFailure]) + extends KnownJobFailureException + with ThrowableAggregation { override def exceptionContext = "Runtime validation failed" override val stderrPath: Option[Path] = None } -final case class JobAlreadyFailedInJobStore(jobTag: String, originalErrorMessage: String) extends KnownJobFailureException { +final case class JobAlreadyFailedInJobStore(jobTag: String, originalErrorMessage: String) + extends KnownJobFailureException { override def stderrPath: Option[Path] = None override def getMessage = originalErrorMessage } diff --git a/backend/src/main/scala/cromwell/backend/async/package.scala b/backend/src/main/scala/cromwell/backend/async/package.scala index b496126ef11..1f7037e711e 100644 --- a/backend/src/main/scala/cromwell/backend/async/package.scala +++ b/backend/src/main/scala/cromwell/backend/async/package.scala @@ -2,7 +2,6 @@ package cromwell.backend import scala.concurrent.{ExecutionContext, Future} - package object async { implicit class EnhancedFutureFuture[A](val ffa: Future[Future[A]])(implicit ec: ExecutionContext) { def flatten: Future[A] = ffa flatMap identity diff --git a/backend/src/main/scala/cromwell/backend/backend.scala b/backend/src/main/scala/cromwell/backend/backend.scala index ea413c10367..8a4453a03f9 100644 --- a/backend/src/main/scala/cromwell/backend/backend.scala +++ b/backend/src/main/scala/cromwell/backend/backend.scala @@ -26,9 +26,7 @@ import scala.util.Try /** * For uniquely identifying a job which has been or will be sent to the backend. */ -case class BackendJobDescriptorKey(call: CommandCallNode, - index: Option[Int], - attempt: Int) extends CallKey { +case class BackendJobDescriptorKey(call: CommandCallNode, index: Option[Int], attempt: Int) extends CallKey { def node = call private val indexString = index map { _.toString } getOrElse "NA" lazy val tag = s"${call.fullyQualifiedName}:$indexString:$attempt" @@ -44,15 +42,19 @@ final case class BackendJobDescriptor(workflowDescriptor: BackendWorkflowDescrip evaluatedTaskInputs: WomEvaluatedCallInputs, maybeCallCachingEligible: MaybeCallCachingEligible, dockerSize: Option[DockerSize], - prefetchedKvStoreEntries: Map[String, KvResponse]) { + prefetchedKvStoreEntries: Map[String, KvResponse] +) { val fullyQualifiedInputs: Map[String, WomValue] = evaluatedTaskInputs map { case (declaration, value) => key.call.identifier.combine(declaration.name).fullyQualifiedName.value -> value } - def findInputFilesByParameterMeta(filter: MetaValueElement => Boolean): Set[WomFile] = evaluatedTaskInputs.collect { - case (declaration, value) if declaration.parameterMeta.exists(filter) => findFiles(value) - }.flatten.toSet + def findInputFilesByParameterMeta(filter: MetaValueElement => Boolean): Set[WomFile] = evaluatedTaskInputs + .collect { + case (declaration, value) if declaration.parameterMeta.exists(filter) => findFiles(value) + } + .flatten + .toSet def findFiles(v: WomValue): Set[WomFile] = v match { case value: WomFile => Set(value) @@ -79,11 +81,13 @@ case class BackendWorkflowDescriptor(id: WorkflowId, customLabels: Labels, hogGroup: HogGroup, breadCrumbs: List[BackendJobBreadCrumb], - outputRuntimeExtractor: Option[WomOutputRuntimeExtractor]) { + outputRuntimeExtractor: Option[WomOutputRuntimeExtractor] +) { val rootWorkflow = breadCrumbs.headOption.map(_.callable).getOrElse(callable) val possiblyNotRootWorkflowId = id.toPossiblyNotRoot val rootWorkflowId = breadCrumbs.headOption.map(_.id).getOrElse(id).toRoot + val possibleParentWorkflowId = breadCrumbs.lastOption.map(_.id) override def toString: String = s"[BackendWorkflowDescriptor id=${id.shortString} workflowName=${callable.name}]" def getWorkflowOption(key: WorkflowOption) = workflowOptions.get(key).toOption @@ -99,33 +103,31 @@ case class BackendConfigurationDescriptor(backendConfig: Config, globalConfig: C Option(backendConfig.getConfig("default-runtime-attributes")) else None - + // So it can be overridden in tests - private [backend] lazy val cromwellFileSystems = CromwellFileSystems.instance + private[backend] lazy val cromwellFileSystems = CromwellFileSystems.instance - lazy val configuredPathBuilderFactories: Map[String, PathBuilderFactory] = { + lazy val configuredPathBuilderFactories: Map[String, PathBuilderFactory] = cromwellFileSystems.factoriesFromConfig(backendConfig).unsafe("Failed to instantiate backend filesystem") - } - private lazy val configuredFactoriesWithDefault = if (configuredPathBuilderFactories.values.exists(_ == DefaultPathBuilderFactory)) { - configuredPathBuilderFactories - } else configuredPathBuilderFactories + DefaultPathBuilderFactory.tuple + private lazy val configuredFactoriesWithDefault = + if (configuredPathBuilderFactories.values.exists(_ == DefaultPathBuilderFactory)) { + configuredPathBuilderFactories + } else configuredPathBuilderFactories + DefaultPathBuilderFactory.tuple /** * Creates path builders using only the configured factories. */ - def pathBuilders(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = { + def pathBuilders(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = PathBuilderFactory.instantiatePathBuilders(configuredPathBuilderFactories.values.toList, workflowOptions) - } /** * Creates path builders using only the configured factories + the default factory */ - def pathBuildersWithDefault(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = { + def pathBuildersWithDefault(workflowOptions: WorkflowOptions)(implicit as: ActorSystem) = PathBuilderFactory.instantiatePathBuilders(configuredFactoriesWithDefault.values.toList, workflowOptions) - } - lazy val slowJobWarningAfter = backendConfig.as[Option[FiniteDuration]](path="slow-job-warning-time") + lazy val slowJobWarningAfter = backendConfig.as[Option[FiniteDuration]](path = "slow-job-warning-time") } object CommonBackendConfigurationAttributes { @@ -136,9 +138,18 @@ object CommonBackendConfigurationAttributes { "default-runtime-attributes.zones", "default-runtime-attributes.continueOnReturnCode", "default-runtime-attributes.cpu", + "default-runtime-attributes.gpuCount", "default-runtime-attributes.noAddress", "default-runtime-attributes.docker", "default-runtime-attributes.queueArn", + "default-runtime-attributes.awsBatchRetryAttempts", + "default-runtime-attributes.maxRetries", + "default-runtime-attributes.awsBatchEvaluateOnExit", + "default-runtime-attributes.sharedMemorySize", + "default-runtime-attributes.ulimits", + "default-runtime-attributes.efsDelocalize", + "default-runtime-attributes.efsMakeMD5", + "default-runtime-attributes.tagResources", "default-runtime-attributes.failOnStderr", "slow-job-warning-time", "dockerhub", @@ -146,10 +157,33 @@ object CommonBackendConfigurationAttributes { "dockerhub.token", "dockerhub.auth", "dockerhub.key-name", - "name-for-call-caching-purposes", + "name-for-call-caching-purposes" ) } final case class AttemptedLookupResult(name: String, value: Try[WomValue]) { def toPair = name -> value } + +sealed trait Platform { + def runtimeKey: String +} + +object Platform { + def all: Seq[Platform] = Seq(Gcp, Azure, Aws) + + def apply(str: String): Option[Platform] = + all.find(_.runtimeKey == str) +} + +object Gcp extends Platform { + override def runtimeKey: String = "gcp" +} + +object Azure extends Platform { + override def runtimeKey: String = "azure" +} + +object Aws extends Platform { + override def runtimeKey: String = "aws" +} diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala index 5c491c5a741..b84843fea5e 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummyAsyncExecutionActor.scala @@ -9,7 +9,12 @@ import cats.implicits._ import common.exception.AggregatedMessageException import common.validation.ErrorOr.ErrorOr import cromwell.backend.BackendJobLifecycleActor -import cromwell.backend.async.{ExecutionHandle, FailedNonRetryableExecutionHandle, PendingExecutionHandle, SuccessfulExecutionHandle} +import cromwell.backend.async.{ + ExecutionHandle, + FailedNonRetryableExecutionHandle, + PendingExecutionHandle, + SuccessfulExecutionHandle +} import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardAsyncExecutionActorParams, StandardAsyncJob} import cromwell.core.CallOutputs import cromwell.core.retry.SimpleExponentialBackoff @@ -22,12 +27,13 @@ import scala.concurrent.Future import scala.concurrent.duration._ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecutionActorParams) - extends BackendJobLifecycleActor + extends BackendJobLifecycleActor with StandardAsyncExecutionActor with CromwellInstrumentation { /** The type of the run info when a job is started. */ override type StandardAsyncRunInfo = String + /** The type of the run status returned during each poll. */ override type StandardAsyncRunState = String @@ -46,14 +52,17 @@ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecuti override def dockerImageUsed: Option[String] = None - override def pollBackOff: SimpleExponentialBackoff = SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) + override def pollBackOff: SimpleExponentialBackoff = + SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) - override def executeOrRecoverBackOff: SimpleExponentialBackoff = SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) + override def executeOrRecoverBackOff: SimpleExponentialBackoff = + SimpleExponentialBackoff(initialInterval = 1.second, maxInterval = 300.seconds, multiplier = 1.1) override val logJobIds: Boolean = false val singletonActor = standardParams.backendSingletonActorOption.getOrElse( - throw new RuntimeException("Dummy Backend actor cannot exist without its singleton actor")) + throw new RuntimeException("Dummy Backend actor cannot exist without its singleton actor") + ) var finishTime: Option[OffsetDateTime] = None @@ -71,46 +80,54 @@ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecuti ) } - override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[String] = { + override def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[String] = finishTime match { - case Some(ft) if (ft.isBefore(OffsetDateTime.now)) => Future.successful("done") + case Some(ft) if ft.isBefore(OffsetDateTime.now) => Future.successful("done") case Some(_) => Future.successful("running") - case None => Future.failed(new Exception("Dummy backend polling for status before finishTime is established(!!?)")) + case None => + Future.failed(new Exception("Dummy backend polling for status before finishTime is established(!!?)")) } - } - - override def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle, state: String): Future[ExecutionHandle] = { - + override def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle, + state: String + ): Future[ExecutionHandle] = if (state == "done") { increment(NonEmptyList("jobs", List("dummy", "executing", "done"))) singletonActor ! DummySingletonActor.MinusOne - val outputsValidation: ErrorOr[Map[OutputPort, WomValue]] = jobDescriptor.taskCall.outputPorts.toList.traverse { - case expressionBasedOutputPort: ExpressionBasedOutputPort => - expressionBasedOutputPort.expression.evaluateValue(Map.empty, NoIoFunctionSet).map(expressionBasedOutputPort -> _) - case other => s"Unknown output port type for Dummy backend output evaluator: ${other.getClass.getSimpleName}".invalidNel - }.map(_.toMap) + val outputsValidation: ErrorOr[Map[OutputPort, WomValue]] = jobDescriptor.taskCall.outputPorts.toList + .traverse { + case expressionBasedOutputPort: ExpressionBasedOutputPort => + expressionBasedOutputPort.expression + .evaluateValue(Map.empty, NoIoFunctionSet) + .map(expressionBasedOutputPort -> _) + case other => + s"Unknown output port type for Dummy backend output evaluator: ${other.getClass.getSimpleName}".invalidNel + } + .map(_.toMap) outputsValidation match { case Valid(outputs) => - Future.successful(SuccessfulExecutionHandle( - outputs = CallOutputs(outputs.toMap), - returnCode = 0, - jobDetritusFiles = Map.empty, - executionEvents = Seq.empty, - resultsClonedFrom = None - )) + Future.successful( + SuccessfulExecutionHandle( + outputs = CallOutputs(outputs.toMap), + returnCode = 0, + jobDetritusFiles = Map.empty, + executionEvents = Seq.empty, + resultsClonedFrom = None + ) + ) case Invalid(errors) => - Future.successful(FailedNonRetryableExecutionHandle( - throwable = AggregatedMessageException("Evaluate outputs from dummy job", errors.toList), - returnCode = None, - kvPairsToSave = None - )) + Future.successful( + FailedNonRetryableExecutionHandle( + throwable = AggregatedMessageException("Evaluate outputs from dummy job", errors.toList), + returnCode = None, + kvPairsToSave = None + ) + ) } - } - else if (state == "running") { + } else if (state == "running") { Future.successful( PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunState]( jobDescriptor = jobDescriptor, @@ -119,9 +136,7 @@ class DummyAsyncExecutionActor(override val standardParams: StandardAsyncExecuti previousState = Option(state) ) ) - } - else { + } else { Future.failed(new Exception(s"Unexpected Dummy state in handlePollSuccess: $state")) } - } } diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala b/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala index 34fd2e8bc51..53a310d1dd8 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummyInitializationActor.scala @@ -2,16 +2,22 @@ package cromwell.backend.dummy import cats.syntax.validated._ import common.validation.ErrorOr.ErrorOr -import cromwell.backend.standard.{StandardInitializationActor, StandardInitializationActorParams, StandardValidatedRuntimeAttributesBuilder} +import cromwell.backend.standard.{ + StandardInitializationActor, + StandardInitializationActorParams, + StandardValidatedRuntimeAttributesBuilder +} import cromwell.backend.validation.RuntimeAttributesValidation import wom.expression.WomExpression import wom.types.{WomStringType, WomType} import wom.values.{WomString, WomValue} class DummyInitializationActor(pipelinesParams: StandardInitializationActorParams) - extends StandardInitializationActor(pipelinesParams) { + extends StandardInitializationActor(pipelinesParams) { - override protected lazy val runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] = Map("backend" -> { _ => true } ) + override protected lazy val runtimeAttributeValidators: Map[String, Option[WomExpression] => Boolean] = Map( + "backend" -> { _ => true } + ) // Specific validator for "backend" to let me specify it in test cases (to avoid accidentally submitting the workflow to real backends!) val backendAttributeValidation: RuntimeAttributesValidation[String] = new RuntimeAttributesValidation[String] { @@ -25,5 +31,6 @@ class DummyInitializationActor(pipelinesParams: StandardInitializationActorParam } } - override def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = super.runtimeAttributesBuilder.withValidation(backendAttributeValidation) + override def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = + super.runtimeAttributesBuilder.withValidation(backendAttributeValidation) } diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala index ee959513a36..492de4970a9 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummyLifecycleActorFactory.scala @@ -3,9 +3,15 @@ package cromwell.backend.dummy import akka.actor.{ActorRef, Props} import cromwell.backend.BackendConfigurationDescriptor import cromwell.backend.standard.callcaching.{StandardCacheHitCopyingActor, StandardFileHashingActor} -import cromwell.backend.standard.{StandardAsyncExecutionActor, StandardInitializationActor, StandardLifecycleActorFactory} +import cromwell.backend.standard.{ + StandardAsyncExecutionActor, + StandardInitializationActor, + StandardLifecycleActorFactory +} -class DummyLifecycleActorFactory(override val name: String, override val configurationDescriptor: BackendConfigurationDescriptor) extends StandardLifecycleActorFactory { +class DummyLifecycleActorFactory(override val name: String, + override val configurationDescriptor: BackendConfigurationDescriptor +) extends StandardLifecycleActorFactory { /** * @return the key to use for storing and looking up the job id. @@ -23,8 +29,11 @@ class DummyLifecycleActorFactory(override val name: String, override val configu // Don't hash files override lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = None - override def backendSingletonActorProps(serviceRegistryActor: ActorRef): Option[Props] = Option(Props(new DummySingletonActor())) + override def backendSingletonActorProps(serviceRegistryActor: ActorRef): Option[Props] = Option( + Props(new DummySingletonActor()) + ) - override lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = classOf[DummyInitializationActor] + override lazy val initializationActorClass: Class[_ <: StandardInitializationActor] = + classOf[DummyInitializationActor] } diff --git a/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala b/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala index 35c689ae2b4..4ea2cf7951c 100644 --- a/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala +++ b/backend/src/main/scala/cromwell/backend/dummy/DummySingletonActor.scala @@ -22,7 +22,7 @@ final class DummySingletonActor() extends Actor with StrictLogging { case PlusOne => count = count + 1 case MinusOne => count = count - 1 case PrintCount => - if(countHistory.lastOption.exists(_._2 != count)) { + if (countHistory.lastOption.exists(_._2 != count)) { countHistory = countHistory :+ (OffsetDateTime.now() -> count) logger.info("The current count is now: " + count) if (count == 0) { @@ -52,7 +52,7 @@ final class DummySingletonActor() extends Actor with StrictLogging { bw.close() } - context.system.scheduler.schedule(10.seconds, 1.second) { self ! PrintCount } + context.system.scheduler.schedule(10.seconds, 1.second)(self ! PrintCount) } object DummySingletonActor { @@ -60,4 +60,3 @@ object DummySingletonActor { case object MinusOne case object PrintCount } - diff --git a/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala b/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala index 7b68e2bf723..84d45d1ee61 100644 --- a/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/io/DirectoryFunctions.scala @@ -11,7 +11,14 @@ import cromwell.core.path.{Path, PathFactory} import wom.expression.IoFunctionSet.{IoDirectory, IoElement, IoFile} import wom.expression.{IoFunctionSet, IoFunctionSetAdapter} import wom.graph.CommandCallNode -import wom.values.{WomFile, WomGlobFile, WomMaybeListedDirectory, WomMaybePopulatedFile, WomSingleFile, WomUnlistedDirectory} +import wom.values.{ + WomFile, + WomGlobFile, + WomMaybeListedDirectory, + WomMaybePopulatedFile, + WomSingleFile, + WomUnlistedDirectory +} import scala.concurrent.Future import scala.util.Try @@ -21,13 +28,18 @@ trait DirectoryFunctions extends IoFunctionSet with PathFactory with AsyncIoFunc private lazy val evaluateFileFunctions = new IoFunctionSetAdapter(this) with FileEvaluationIoFunctionSet def findDirectoryOutputs(call: CommandCallNode, - jobDescriptor: BackendJobDescriptor): ErrorOr[List[WomUnlistedDirectory]] = { + jobDescriptor: BackendJobDescriptor + ): ErrorOr[List[WomUnlistedDirectory]] = call.callable.outputs.flatTraverse[ErrorOr, WomUnlistedDirectory] { outputDefinition => - outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, evaluateFileFunctions, outputDefinition.womType) map { - _.toList.flatMap(_.file.flattenFiles) collect { case unlistedDirectory: WomUnlistedDirectory => unlistedDirectory } + outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, + evaluateFileFunctions, + outputDefinition.womType + ) map { + _.toList.flatMap(_.file.flattenFiles) collect { case unlistedDirectory: WomUnlistedDirectory => + unlistedDirectory + } } } - } override def isDirectory(path: String) = asyncIo.isDirectory(buildPath(path)) @@ -39,7 +51,7 @@ trait DirectoryFunctions extends IoFunctionSet with PathFactory with AsyncIoFunc * implementation which lists files and directories children. What we need is the unix behavior, even for cloud filesystems. * 3) It uses the isDirectory function directly on the path, which cannot be trusted for GCS paths. It should use asyncIo.isDirectory instead. */ - override def listDirectory(path: String)(visited: Vector[String] = Vector.empty): Future[Iterator[IoElement]] = { + override def listDirectory(path: String)(visited: Vector[String] = Vector.empty): Future[Iterator[IoElement]] = Future.fromTry(Try { val visitedPaths = visited.map(buildPath) val cromwellPath = buildPath(path.ensureSlashed) @@ -47,21 +59,21 @@ trait DirectoryFunctions extends IoFunctionSet with PathFactory with AsyncIoFunc // To prevent infinite recursion through symbolic links make sure we don't visit the same directory twice def hasBeenVisited(other: Path) = visitedPaths.exists(_.isSameFileAs(other)) - cromwellPath.list.collect({ - case directory if directory.isDirectory && - !cromwellPath.isSamePathAs(directory) && - !hasBeenVisited(directory) => IoDirectory(directory.pathAsString) + cromwellPath.list.collect { + case directory + if directory.isDirectory && + !cromwellPath.isSamePathAs(directory) && + !hasBeenVisited(directory) => + IoDirectory(directory.pathAsString) case file => IoFile(file.pathAsString) - }) + } }) - } - override def listAllFilesUnderDirectory(dirPath: String): Future[Seq[String]] = { + override def listAllFilesUnderDirectory(dirPath: String): Future[Seq[String]] = temporaryImplListPaths(dirPath) - } // TODO: WOM: WOMFILE: This will likely use a Tuple2(tar file, dir list file) for each dirPath. - private final def temporaryImplListPaths(dirPath: String): Future[Seq[String]] = { + final private def temporaryImplListPaths(dirPath: String): Future[Seq[String]] = { val errorOrPaths = for { dir <- validate(buildPath(dirPath.ensureSlashed)) files <- listFiles(dir) @@ -74,7 +86,7 @@ object DirectoryFunctions { def listFiles(path: Path): ErrorOr[List[Path]] = path.listRecursively.filterNot(_.isDirectory).toList.validNel def listWomSingleFiles(womFile: WomFile, pathFactory: PathFactory): ErrorOr[List[WomSingleFile]] = { - def listWomSingleFiles(womFile: WomFile): ErrorOr[List[WomSingleFile]] = { + def listWomSingleFiles(womFile: WomFile): ErrorOr[List[WomSingleFile]] = womFile match { case womSingleFile: WomSingleFile => List(womSingleFile).valid @@ -99,7 +111,6 @@ object DirectoryFunctions { case _: WomGlobFile => s"Unexpected glob / unable to list glob files at this time: $womFile".invalidNel } - } listWomSingleFiles(womFile) } diff --git a/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala b/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala index f46ff9ce9e2..0408c590cb2 100644 --- a/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala +++ b/backend/src/main/scala/cromwell/backend/io/FileEvaluationIoFunctionSet.scala @@ -5,5 +5,6 @@ import wom.expression.IoFunctionSet import scala.concurrent.Future trait FileEvaluationIoFunctionSet { this: IoFunctionSet => - override def glob(pattern: String) = Future.failed(new IllegalStateException("Cannot perform globing while evaluating files")) + override def glob(pattern: String) = + Future.failed(new IllegalStateException("Cannot perform globing while evaluating files")) } diff --git a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala index 6f0ae823bd8..5ed3b4a30ff 100644 --- a/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/io/GlobFunctions.scala @@ -19,7 +19,10 @@ trait GlobFunctions extends IoFunctionSet with AsyncIoFunctions { def findGlobOutputs(call: CommandCallNode, jobDescriptor: BackendJobDescriptor): ErrorOr[List[WomGlobFile]] = { def fromOutputs = call.callable.outputs.flatTraverse[ErrorOr, WomGlobFile] { outputDefinition => - outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, evaluateFileFunctions, outputDefinition.womType) map { + outputDefinition.expression.evaluateFiles(jobDescriptor.localInputs, + evaluateFileFunctions, + outputDefinition.womType + ) map { _.toList.flatMap(_.file.flattenFiles) collect { case glob: WomGlobFile => glob } } } @@ -38,9 +41,9 @@ trait GlobFunctions extends IoFunctionSet with AsyncIoFunctions { import GlobFunctions._ val globPatternName = globName(pattern) val listFilePath = callContext.root.resolve(s"${globName(pattern)}.list") - asyncIo.readLinesAsync(listFilePath.toRealPath()) map { lines => + asyncIo.readLinesAsync(listFilePath.getSymlinkSafePath()) map { lines => lines.toList map { fileName => - (callContext.root / globPatternName / fileName).pathAsString + (callContext.root / globPatternName / fileName).pathAsString } } } diff --git a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala index eb9e9ec31d7..05ad6a56dc1 100644 --- a/backend/src/main/scala/cromwell/backend/io/JobPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/JobPaths.scala @@ -54,16 +54,14 @@ trait JobPaths { /** * Return a host path corresponding to the specified container path. */ - def hostPathFromContainerPath(string: String): Path = { + def hostPathFromContainerPath(string: String): Path = // No container here, just return a Path of the absolute path to the file. callExecutionRoot.resolve(string.stripPrefix(rootWithSlash)) - } def hostPathFromContainerInputs(string: String): Path = // No container here, just return a Path of the absolute path to the file. callExecutionRoot.resolve(string.stripPrefix(rootWithSlash)) - def scriptFilename: String = "script" def dockerCidFilename: String = "docker_cid" diff --git a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala index f6d8855f3fa..2fb81a44019 100644 --- a/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala +++ b/backend/src/main/scala/cromwell/backend/io/JobPathsWithDocker.scala @@ -6,18 +6,19 @@ import cromwell.backend.{BackendJobDescriptorKey, BackendWorkflowDescriptor} import cromwell.core.path.Path object JobPathsWithDocker { - def apply(jobKey: BackendJobDescriptorKey, - workflowDescriptor: BackendWorkflowDescriptor, - config: Config) = { + def apply(jobKey: BackendJobDescriptorKey, workflowDescriptor: BackendWorkflowDescriptor, config: Config) = { val workflowPaths = new WorkflowPathsWithDocker(workflowDescriptor, config, WorkflowPaths.DefaultPathBuilders) new JobPathsWithDocker(workflowPaths, jobKey) } } -case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowPathsWithDocker, jobKey: BackendJobDescriptorKey, override val isCallCacheCopyAttempt: Boolean = false) extends JobPaths { +case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowPathsWithDocker, + jobKey: BackendJobDescriptorKey, + override val isCallCacheCopyAttempt: Boolean = false +) extends JobPaths { import JobPaths._ - override lazy val callExecutionRoot = { callRoot.resolve("execution") } + override lazy val callExecutionRoot = callRoot.resolve("execution") override def isDocker: Boolean = true val callDockerRoot = callPathBuilder(workflowPaths.dockerWorkflowRoot, jobKey, isCallCacheCopyAttempt) val callExecutionDockerRoot = callDockerRoot.resolve("execution") @@ -29,34 +30,31 @@ case class JobPathsWithDocker private[io] (override val workflowPaths: WorkflowP override def isInExecution(string: String): Boolean = string.startsWith(callExecutionDockerRootWithSlash) - override def hostPathFromContainerPath(string: String): Path = { + override def hostPathFromContainerPath(string: String): Path = callExecutionRoot.resolve(string.stripPrefix(callExecutionDockerRootWithSlash)) - } - override def hostPathFromContainerInputs(string: String): Path = { val stripped = string.stripPrefix(callInputsDockerRootWithSlash) callInputsRoot.resolve(stripped) } - def toDockerPath(path: Path): Path = { + def toDockerPath(path: Path): Path = path.toAbsolutePath match { case p if p.startsWith(workflowPaths.dockerRoot) => p case p => /* For example: - * - * p = /abs/path/to/cromwell-executions/three-step/f00ba4/call-ps/stdout.txt - * localExecutionRoot = /abs/path/to/cromwell-executions - * subpath = three-step/f00ba4/call-ps/stdout.txt - * - * return value = /root/three-step/f00ba4/call-ps/stdout.txt - * - * TODO: this assumes that p.startsWith(localExecutionRoot) - */ + * + * p = /abs/path/to/cromwell-executions/three-step/f00ba4/call-ps/stdout.txt + * localExecutionRoot = /abs/path/to/cromwell-executions + * subpath = three-step/f00ba4/call-ps/stdout.txt + * + * return value = /root/three-step/f00ba4/call-ps/stdout.txt + * + * TODO: this assumes that p.startsWith(localExecutionRoot) + */ val subpath = p.subpath(workflowPaths.executionRoot.getNameCount, p.getNameCount) workflowPaths.dockerRoot.resolve(subpath) } - } override def forCallCacheCopyAttempts: JobPaths = this.copy(isCallCacheCopyAttempt = true) } diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala index 8c233717898..228285dbd5d 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPaths.scala @@ -20,7 +20,8 @@ trait WorkflowPaths extends PathFactory { /** * Path (as a String) of the root directory Cromwell should use for ALL workflows. */ - protected lazy val executionRootString: String = config.as[Option[String]]("root").getOrElse(WorkflowPaths.DefaultExecutionRootString) + protected lazy val executionRootString: String = + config.as[Option[String]]("root").getOrElse(WorkflowPaths.DefaultExecutionRootString) /** * Implementers of this trait might override this to provide an appropriate prefix corresponding to the execution root @@ -51,18 +52,17 @@ trait WorkflowPaths extends PathFactory { def getPath(url: String): Try[Path] = Try(buildPath(url)) // Rebuild potential intermediate call directories in case of a sub workflow - protected def workflowPathBuilder(root: Path): Path = { - workflowDescriptor.breadCrumbs.foldLeft(root)((acc, breadCrumb) => { - breadCrumb.toPath(acc) - }).resolve(workflowDescriptor.callable.name).resolve(workflowDescriptor.id.toString + "/") - } + protected def workflowPathBuilder(root: Path): Path = + workflowDescriptor.breadCrumbs + .foldLeft(root)((acc, breadCrumb) => breadCrumb.toPath(acc)) + .resolve(workflowDescriptor.callable.name) + .resolve(workflowDescriptor.id.toString + "/") lazy val finalCallLogsPath: Option[Path] = workflowDescriptor.getWorkflowOption(FinalCallLogsDir) map getPath map { _.get } - def toJobPaths(jobDescriptor: BackendJobDescriptor): JobPaths = { + def toJobPaths(jobDescriptor: BackendJobDescriptor): JobPaths = toJobPaths(jobDescriptor.key, jobDescriptor.workflowDescriptor) - } /** * Creates job paths using the key and workflow descriptor. @@ -73,11 +73,10 @@ trait WorkflowPaths extends PathFactory { * @param jobWorkflowDescriptor The workflow descriptor for the job. * @return The paths for the job. */ - def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths = { + def toJobPaths(jobKey: BackendJobDescriptorKey, jobWorkflowDescriptor: BackendWorkflowDescriptor): JobPaths = // If the descriptors are the same, no need to create a new WorkflowPaths if (workflowDescriptor == jobWorkflowDescriptor) toJobPaths(this, jobKey) else toJobPaths(withDescriptor(jobWorkflowDescriptor), jobKey) - } protected def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPaths diff --git a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala index 78dc1ed77e3..38ea17fe61c 100644 --- a/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala +++ b/backend/src/main/scala/cromwell/backend/io/WorkflowPathsWithDocker.scala @@ -9,16 +9,19 @@ object WorkflowPathsWithDocker { val DefaultDockerRoot = "/cromwell-executions" } -final case class WorkflowPathsWithDocker(workflowDescriptor: BackendWorkflowDescriptor, config: Config, pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders) extends WorkflowPaths { +final case class WorkflowPathsWithDocker(workflowDescriptor: BackendWorkflowDescriptor, + config: Config, + pathBuilders: List[PathBuilder] = WorkflowPaths.DefaultPathBuilders +) extends WorkflowPaths { val dockerRoot: Path = DefaultPathBuilder.get( config.getOrElse[String]("dockerRoot", WorkflowPathsWithDocker.DefaultDockerRoot) ) val dockerWorkflowRoot: Path = workflowPathBuilder(dockerRoot) - override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPathsWithDocker = { + override def toJobPaths(workflowPaths: WorkflowPaths, jobKey: BackendJobDescriptorKey): JobPathsWithDocker = new JobPathsWithDocker(workflowPaths.asInstanceOf[WorkflowPathsWithDocker], jobKey, isCallCacheCopyAttempt = false) - } - override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = this.copy(workflowDescriptor = workflowDescriptor) + override protected def withDescriptor(workflowDescriptor: BackendWorkflowDescriptor): WorkflowPaths = + this.copy(workflowDescriptor = workflowDescriptor) } diff --git a/backend/src/main/scala/cromwell/backend/package.scala b/backend/src/main/scala/cromwell/backend/package.scala index 132fcf57887..ca58701641f 100644 --- a/backend/src/main/scala/cromwell/backend/package.scala +++ b/backend/src/main/scala/cromwell/backend/package.scala @@ -1,6 +1,7 @@ package cromwell package object backend { + /** Represents the jobKeys executed by a (potentially sub-) workflow at a given point in time */ type JobExecutionMap = Map[BackendWorkflowDescriptor, List[BackendJobDescriptorKey]] } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala index 0f1e2c931a8..1a7e2ceae4e 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardAsyncExecutionActor.scala @@ -1,7 +1,6 @@ package cromwell.backend.standard import java.io.IOException - import akka.actor.{Actor, ActorLogging, ActorRef} import akka.event.LoggingReceive import cats.implicits._ @@ -12,7 +11,11 @@ import common.util.TryUtil import common.validation.ErrorOr.{ErrorOr, ShortCircuitingFlatMap} import common.validation.IOChecked._ import common.validation.Validation._ -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobAbortedResponse, JobReconnectionNotSupportedException} +import cromwell.backend.BackendJobExecutionActor.{ + BackendJobExecutionResponse, + JobAbortedResponse, + JobReconnectionNotSupportedException +} import cromwell.backend.BackendLifecycleActor.AbortJobCommand import cromwell.backend.BackendLifecycleActorFactory.{FailedRetryCountKey, MemoryMultiplierKey} import cromwell.backend.OutputEvaluator._ @@ -34,23 +37,23 @@ import net.ceedubs.ficus.Ficus._ import org.apache.commons.lang3.StringUtils import org.apache.commons.lang3.exception.ExceptionUtils import shapeless.Coproduct -import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression, RuntimeEnvironment} +import wom.callable.{AdHocValue, CommandTaskDefinition, ContainerizedInputExpression} import wom.expression.WomExpression import wom.graph.LocalName import wom.values._ import wom.{CommandSetupSideEffectFile, InstantiatedCommand, WomFileMapper} - +import java.net.URLDecoder import scala.concurrent._ import scala.concurrent.duration._ import scala.util.{Failure, Success, Try} trait StandardAsyncExecutionActorParams extends StandardJobExecutionActorParams { + /** The promise that will be completed when the async run is complete. */ def completionPromise: Promise[BackendJobExecutionResponse] } -case class DefaultStandardAsyncExecutionActorParams -( +case class DefaultStandardAsyncExecutionActorParams( override val jobIdKey: String, override val serviceRegistryActor: ActorRef, override val ioActor: ActorRef, @@ -62,6 +65,10 @@ case class DefaultStandardAsyncExecutionActorParams override val minimumRuntimeSettings: MinimumRuntimeSettings ) extends StandardAsyncExecutionActorParams +// Typically we want to "executeInSubshell" for encapsulation of bash code. +// Override to `false` when we need the script to set an environment variable in the parent shell. +case class ScriptPreambleData(bashString: String, executeInSubshell: Boolean = true) + /** * An extension of the generic AsyncBackendJobExecutionActor providing a standard abstract implementation of an * asynchronous polling backend. @@ -73,7 +80,11 @@ case class DefaultStandardAsyncExecutionActorParams * as the common behavior among the backends adjusts in unison. */ trait StandardAsyncExecutionActor - extends AsyncBackendJobExecutionActor with StandardCachingActorHelper with AsyncIoActorClient with KvClient with SlowJobWarning { + extends AsyncBackendJobExecutionActor + with StandardCachingActorHelper + with AsyncIoActorClient + with KvClient + with SlowJobWarning { this: Actor with ActorLogging with BackendJobLifecycleActor => override lazy val ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder @@ -97,7 +108,8 @@ trait StandardAsyncExecutionActor def statusEquivalentTo(thiz: StandardAsyncRunState)(that: StandardAsyncRunState): Boolean /** The pending execution handle for each poll. */ - type StandardAsyncPendingExecutionHandle = PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunState] + type StandardAsyncPendingExecutionHandle = + PendingExecutionHandle[StandardAsyncJob, StandardAsyncRunInfo, StandardAsyncRunState] /** Standard set of parameters passed to the backend. */ def standardParams: StandardAsyncExecutionActorParams @@ -129,7 +141,7 @@ trait StandardAsyncExecutionActor lazy val temporaryDirectory: String = configurationDescriptor.backendConfig.getOrElse( path = "temporary-directory", - default = s"""$$(mkdir -p "${runtimeEnvironment.tempPath}" && echo "${runtimeEnvironment.tempPath}")""" + default = """$(mktemp -d "$PWD"/tmp.XXXXXX)""" ) val logJobIds: Boolean = true @@ -147,16 +159,14 @@ trait StandardAsyncExecutionActor protected def cloudResolveWomFile(womFile: WomFile): WomFile = womFile /** Process files while resolving files that will not be localized to their actual cloud locations. */ - private def mapOrCloudResolve(mapper: WomFile => WomFile): WomValue => Try[WomValue] = { - WomFileMapper.mapWomFiles( - womFile => - if (inputsToNotLocalize.contains(womFile)) { - cloudResolveWomFile(womFile) - } else { - mapper(womFile) - } + private def mapOrCloudResolve(mapper: WomFile => WomFile): WomValue => Try[WomValue] = + WomFileMapper.mapWomFiles(womFile => + if (inputsToNotLocalize.contains(womFile)) { + cloudResolveWomFile(womFile) + } else { + mapper(womFile) + } ) - } /** Process files while ignoring files that will not be localized. */ private def mapOrNoResolve(mapper: WomFile => WomFile): WomValue => Try[WomValue] = { @@ -165,7 +175,12 @@ trait StandardAsyncExecutionActor if (inputsToNotLocalize.contains(womFile)) { womFile } else { - mapper(womFile) + // resolve + val mapped_wom = mapper(womFile) + // decode url encoded values generated by the mapper + val decoded_womValue = URLDecoder.decode(mapped_wom.valueString,"UTF-8") + // convert to womfile again + WomFile(womFile.womFileType, decoded_womValue) } ) } @@ -173,13 +188,14 @@ trait StandardAsyncExecutionActor /** @see [[Command.instantiate]] */ final def commandLinePreProcessor(inputs: WomEvaluatedCallInputs): Try[WomEvaluatedCallInputs] = { val map = inputs map { case (k, v) => k -> mapOrCloudResolve(preProcessWomFile)(v) } - TryUtil.sequenceMap(map). - recoverWith { - case e => Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker) - } + TryUtil.sequenceMap(map).recoverWith { case e => + Failure(new IOException(e.getMessage) with CromwellFatalExceptionMarker) + } } - final lazy val localizedInputs: Try[WomEvaluatedCallInputs] = commandLinePreProcessor(jobDescriptor.evaluatedTaskInputs) + final lazy val localizedInputs: Try[WomEvaluatedCallInputs] = commandLinePreProcessor( + jobDescriptor.evaluatedTaskInputs + ) /** * Maps WomFile to a local path, for use in the commandLineValueMapper. @@ -202,48 +218,62 @@ trait StandardAsyncExecutionActor def inputsToNotLocalize: Set[WomFile] = Set.empty /** @see [[Command.instantiate]] */ - final lazy val commandLineValueMapper: WomValue => WomValue = { - womValue => mapOrNoResolve(mapCommandLineWomFile)(womValue).get + final lazy val commandLineValueMapper: WomValue => WomValue = { womValue => + mapOrNoResolve(mapCommandLineWomFile)(womValue).get } /** @see [[Command.instantiate]] */ - final lazy val commandLineJobInputValueMapper: WomValue => WomValue = { - womValue => mapOrNoResolve(mapCommandLineJobInputWomFile)(womValue).get + final lazy val commandLineJobInputValueMapper: WomValue => WomValue = { womValue => + mapOrNoResolve(mapCommandLineJobInputWomFile)(womValue).get } - lazy val jobShell: String = configurationDescriptor.backendConfig.getOrElse("job-shell", - configurationDescriptor.globalConfig.getOrElse("system.job-shell", "/bin/bash")) + lazy val jobShell: String = configurationDescriptor.backendConfig.getOrElse( + "job-shell", + configurationDescriptor.globalConfig.getOrElse("system.job-shell", "/bin/bash") + ) - lazy val abbreviateCommandLength: Int = configurationDescriptor.backendConfig.getOrElse("abbreviate-command-length", - configurationDescriptor.globalConfig.getOrElse("system.abbreviate-command-length", 0)) + lazy val abbreviateCommandLength: Int = configurationDescriptor.backendConfig.getOrElse( + "abbreviate-command-length", + configurationDescriptor.globalConfig.getOrElse("system.abbreviate-command-length", 0) + ) /** * The local path where the command will run. */ lazy val commandDirectory: Path = jobPaths.callExecutionRoot - lazy val memoryRetryErrorKeys: Option[List[String]] = configurationDescriptor.globalConfig.as[Option[List[String]]]("system.memory-retry-error-keys") + lazy val memoryRetryErrorKeys: Option[List[String]] = + configurationDescriptor.globalConfig.as[Option[List[String]]]("system.memory-retry-error-keys") lazy val memoryRetryFactor: Option[MemoryRetryMultiplierRefined] = { jobDescriptor.workflowDescriptor.getWorkflowOption(WorkflowOptions.MemoryRetryMultiplier) flatMap { value: String => Try(value.toDouble) match { - case Success(v) => refineV[MemoryRetryMultiplier](v.toDouble) match { - case Left(e) => - // should not happen, this case should have been screened for and fast-failed during workflow materialization. - log.error(e, s"Programmer error: unexpected failure attempting to read value for workflow option " + - s"'${WorkflowOptions.MemoryRetryMultiplier.name}'. Expected value should be in range 1.0 ≤ n ≤ 99.0") - None - case Right(refined) => Option(refined) - } + case Success(v) => + refineV[MemoryRetryMultiplier](v.toDouble) match { + case Left(e) => + // should not happen, this case should have been screened for and fast-failed during workflow materialization. + log.error( + e, + s"Programmer error: unexpected failure attempting to read value for workflow option " + + s"'${WorkflowOptions.MemoryRetryMultiplier.name}'. Expected value should be in range 1.0 ≤ n ≤ 99.0" + ) + None + case Right(refined) => Option(refined) + } case Failure(e) => // should not happen, this case should have been screened for and fast-failed during workflow materialization. - log.error(e, s"Programmer error: unexpected failure attempting to convert value for workflow option " + - s"'${WorkflowOptions.MemoryRetryMultiplier.name}' to Double.") + log.error( + e, + s"Programmer error: unexpected failure attempting to convert value for workflow option " + + s"'${WorkflowOptions.MemoryRetryMultiplier.name}' to Double." + ) None } } } + lazy val memoryRetryRequested: Boolean = memoryRetryFactor.nonEmpty + /** * Returns the shell scripting for finding all files listed within a directory. * @@ -300,7 +330,8 @@ trait StandardAsyncExecutionActor val globList = parentDirectory./(s"$globDir.list") val controlFileName = "cromwell_glob_control_file" val absoluteGlobValue = commandDirectory.resolve(globFile.value).pathAsString - val globLinkCommand: String = configurationDescriptor.backendConfig.getAs[String]("glob-link-command") + val globLinkCommand: String = configurationDescriptor.backendConfig + .getAs[String]("glob-link-command") .map("( " + _ + " )") .getOrElse("( ln -L GLOB_PATTERN GLOB_DIRECTORY 2> /dev/null ) || ( ln GLOB_PATTERN GLOB_DIRECTORY )") .replaceAll("GLOB_PATTERN", absoluteGlobValue) @@ -327,7 +358,10 @@ trait StandardAsyncExecutionActor } /** Any custom code that should be run within commandScriptContents before the instantiated command. */ - def scriptPreamble: String = "" + def scriptPreamble: ErrorOr[ScriptPreambleData] = ScriptPreambleData("").valid + + /** Any custom code that should be run within commandScriptContents right before exiting. */ + def scriptClosure: String = "" def cwd: Path = commandDirectory def rcPath: Path = cwd./(jobPaths.returnCodeFilename) @@ -345,13 +379,14 @@ trait StandardAsyncExecutionActor // Absolutize any redirect and overridden paths. All of these files must have absolute paths since the command script // references them outside a (cd "execution dir"; ...) subshell. The default names are known to be relative paths, // the names from redirections may or may not be relative. - private def absolutizeContainerPath(path: String): String = { + private def absolutizeContainerPath(path: String): String = if (path.startsWith(cwd.pathAsString)) path else cwd.resolve(path).pathAsString - } def executionStdin: Option[String] = instantiatedCommand.evaluatedStdinRedirection map absolutizeContainerPath - def executionStdout: String = instantiatedCommand.evaluatedStdoutOverride.getOrElse(jobPaths.defaultStdoutFilename) |> absolutizeContainerPath - def executionStderr: String = instantiatedCommand.evaluatedStderrOverride.getOrElse(jobPaths.defaultStderrFilename) |> absolutizeContainerPath + def executionStdout: String = + instantiatedCommand.evaluatedStdoutOverride.getOrElse(jobPaths.defaultStdoutFilename) |> absolutizeContainerPath + def executionStderr: String = + instantiatedCommand.evaluatedStderrOverride.getOrElse(jobPaths.defaultStderrFilename) |> absolutizeContainerPath /* * Ensures the standard paths are correct w.r.t overridden paths. This is called in two places: when generating the command and @@ -361,7 +396,7 @@ trait StandardAsyncExecutionActor * to re-do this before sending the response. */ private var jobPathsUpdated: Boolean = false - private def updateJobPaths(): Unit = if (!jobPathsUpdated) { + def updateJobPaths(): Unit = if (!jobPathsUpdated) { // .get's are safe on stdout and stderr after falling back to default names above. jobPaths.standardPaths = StandardPaths( output = hostPathFromContainerPath(executionStdout), @@ -382,7 +417,7 @@ trait StandardAsyncExecutionActor def commandScriptContents: ErrorOr[String] = { val commandString = instantiatedCommand.commandString val commandStringAbbreviated = StringUtils.abbreviateMiddle(commandString, "...", abbreviateCommandLength) - jobLogger.info(s"`$commandStringAbbreviated`") + jobLogger.debug(s"`$commandStringAbbreviated`") tellMetadata(Map(CallMetadataKeys.CommandLine -> commandStringAbbreviated)) val cwd = commandDirectory @@ -403,9 +438,10 @@ trait StandardAsyncExecutionActor val errorOrGlobFiles: ErrorOr[List[WomGlobFile]] = backendEngineFunctions.findGlobOutputs(call, jobDescriptor) - lazy val environmentVariables = instantiatedCommand.environmentVariables map { case (k, v) => s"""export $k="$v"""" } mkString("", "\n", "\n") + lazy val environmentVariables = instantiatedCommand.environmentVariables map { case (k, v) => + s"""export $k="$v"""" + } mkString ("", "\n", "\n") - val home = jobDescriptor.taskCall.callable.homeOverride.map { _ (runtimeEnvironment) }.getOrElse("$HOME") val shortId = jobDescriptor.workflowDescriptor.id.shortString // Give the out and error FIFO variables names that are unlikely to conflict with anything the user is doing. val (out, err) = (s"out$shortId", s"err$shortId") @@ -417,65 +453,76 @@ trait StandardAsyncExecutionActor // Only adjust the temporary directory permissions if this is executing under Docker. val tmpDirPermissionsAdjustment = if (isDockerRun) s"""chmod 777 "$$tmpDir"""" else "" - val emptyDirectoryFillCommand: String = configurationDescriptor.backendConfig.getAs[String]("empty-dir-fill-command") - .getOrElse( - s"""( - |# add a .file in every empty directory to facilitate directory delocalization on the cloud - |cd $cwd - |find . -type d -exec sh -c '[ -z "$$(ls -A '"'"'{}'"'"')" ] && touch '"'"'{}'"'"'/.file' \\; - |)""".stripMargin) + val emptyDirectoryFillCommand: String = configurationDescriptor.backendConfig + .getAs[String]("empty-dir-fill-command") + .getOrElse(s"""( + |# add a .file in every empty directory to facilitate directory delocalization on the cloud + |cd $cwd + |find . -type d -exec sh -c '[ -z "$$(ls -A '"'"'{}'"'"')" ] && touch '"'"'{}'"'"'/.file' \\; + |)""".stripMargin) + + val errorOrPreamble: ErrorOr[String] = scriptPreamble.map { preambleData => + preambleData.executeInSubshell match { + case true => + s""" + |( + |cd ${cwd.pathAsString} + |${preambleData.bashString} + |) + |""".stripMargin + case false => + s""" + |cd ${cwd.pathAsString} + |${preambleData.bashString} + |""".stripMargin + } + } // The `tee` trickery below is to be able to redirect to known filenames for CWL while also streaming // stdout and stderr for PAPI to periodically upload to cloud storage. // https://stackoverflow.com/questions/692000/how-do-i-write-stderr-to-a-file-while-using-tee-with-a-pipe - (errorOrDirectoryOutputs, errorOrGlobFiles).mapN((directoryOutputs, globFiles) => - s"""|#!$jobShell - |DOCKER_OUTPUT_DIR_LINK - |cd ${cwd.pathAsString} - |tmpDir=$temporaryDirectory - |$tmpDirPermissionsAdjustment - |export _JAVA_OPTIONS=-Djava.io.tmpdir="$$tmpDir" - |export TMPDIR="$$tmpDir" - |export HOME="$home" - |( - |cd ${cwd.pathAsString} - |SCRIPT_PREAMBLE - |) - |$out="$${tmpDir}/out.$$$$" $err="$${tmpDir}/err.$$$$" - |mkfifo "$$$out" "$$$err" - |trap 'rm "$$$out" "$$$err"' EXIT - |touch $stdoutRedirection $stderrRedirection - |tee $stdoutRedirection < "$$$out" & - |tee $stderrRedirection < "$$$err" >&2 & - |( - |cd ${cwd.pathAsString} - |ENVIRONMENT_VARIABLES - |INSTANTIATED_COMMAND - |) $stdinRedirection > "$$$out" 2> "$$$err" - |echo $$? > $rcTmpPath - |$emptyDirectoryFillCommand - |( - |cd ${cwd.pathAsString} - |SCRIPT_EPILOGUE - |${globScripts(globFiles)} - |${directoryScripts(directoryOutputs)} - |) - |mv $rcTmpPath $rcPath - |""".stripMargin - .replace("SCRIPT_PREAMBLE", scriptPreamble) - .replace("ENVIRONMENT_VARIABLES", environmentVariables) - .replace("INSTANTIATED_COMMAND", commandString) - .replace("SCRIPT_EPILOGUE", scriptEpilogue) - .replace("DOCKER_OUTPUT_DIR_LINK", dockerOutputDir)) - } - - def runtimeEnvironmentPathMapper(env: RuntimeEnvironment): RuntimeEnvironment = { - def localize(path: String): String = (WomSingleFile(path) |> commandLineValueMapper).valueString - env.copy(outputPath = env.outputPath |> localize, tempPath = env.tempPath |> localize) - } - - lazy val runtimeEnvironment: RuntimeEnvironment = { - RuntimeEnvironmentBuilder(jobDescriptor.runtimeAttributes, jobPaths)(standardParams.minimumRuntimeSettings) |> runtimeEnvironmentPathMapper + (errorOrDirectoryOutputs, errorOrGlobFiles, errorOrPreamble).mapN((directoryOutputs, globFiles, preamble) => + s"""|#!$jobShell + |DOCKER_OUTPUT_DIR_LINK + |cd ${cwd.pathAsString} + |tmpDir=$temporaryDirectory + |$tmpDirPermissionsAdjustment + |export _JAVA_OPTIONS=-Djava.io.tmpdir="$$tmpDir" + |export TMPDIR="$$tmpDir" + | + |SCRIPT_PREAMBLE + | + |$out="$${tmpDir}/out.$$$$" $err="$${tmpDir}/err.$$$$" + |mkfifo "$$$out" "$$$err" + |trap 'rm "$$$out" "$$$err"' EXIT + |touch $stdoutRedirection $stderrRedirection + |tee $stdoutRedirection < "$$$out" & + |tee $stderrRedirection < "$$$err" >&2 & + |set -x + |( + |cd ${cwd.pathAsString} + |ENVIRONMENT_VARIABLES + |INSTANTIATED_COMMAND + |) $stdinRedirection > "$$$out" 2> "$$$err" + |echo $$? > $rcTmpPath + |set +x + |$emptyDirectoryFillCommand + |( + |cd ${cwd.pathAsString} + |SCRIPT_EPILOGUE + |${globScripts(globFiles)} + |${directoryScripts(directoryOutputs)} + |) + |mv $rcTmpPath $rcPath + |SCRIPT_CLOSURE + |""".stripMargin + .replace("SCRIPT_PREAMBLE", preamble) + .replace("ENVIRONMENT_VARIABLES", environmentVariables) + .replace("INSTANTIATED_COMMAND", commandString) + .replace("SCRIPT_EPILOGUE", scriptEpilogue) + .replace("DOCKER_OUTPUT_DIR_LINK", dockerOutputDir) + .replace("SCRIPT_CLOSURE", scriptClosure) + ) } /** @@ -486,23 +533,21 @@ trait StandardAsyncExecutionActor * relativeLocalizationPath("gs://some/bucket/foo.txt") -> "some/bucket/foo.txt" * etc */ - protected def relativeLocalizationPath(file: WomFile): WomFile = { + protected def relativeLocalizationPath(file: WomFile): WomFile = file.mapFile(value => getPath(value) match { case Success(path) => path.pathWithoutScheme case _ => value } ) - } - protected def fileName(file: WomFile): WomFile = { + protected def fileName(file: WomFile): WomFile = file.mapFile(value => getPath(value) match { case Success(path) => path.name case _ => value } ) - } protected def localizationPath(f: CommandSetupSideEffectFile): WomFile = { val fileTransformer = if (isAdHocFile(f.file)) fileName _ else relativeLocalizationPath _ @@ -518,7 +563,6 @@ trait StandardAsyncExecutionActor * Maybe this should be the other way around: the default implementation is noop and SFS / TES override it ? */ lazy val localizeAdHocValues: List[AdHocValue] => ErrorOr[List[StandardAdHocValue]] = { adHocValues => - // Localize an adhoc file to the callExecutionRoot as needed val localize: (AdHocValue, Path) => Future[LocalizedAdHocValue] = { (adHocValue, file) => val actualName = adHocValue.alternativeName.getOrElse(file.name) @@ -526,17 +570,19 @@ trait StandardAsyncExecutionActor // First check that it's not already there under execution root asyncIo.existsAsync(finalPath) flatMap { // If it's not then copy it - case false => asyncIo.copyAsync(file, finalPath) as { LocalizedAdHocValue(adHocValue, finalPath) } + case false => asyncIo.copyAsync(file, finalPath) as LocalizedAdHocValue(adHocValue, finalPath) case true => Future.successful(LocalizedAdHocValue(adHocValue, finalPath)) } } - adHocValues.traverse[ErrorOr, (AdHocValue, Path)]({ adHocValue => - // Build an actionable Path from the ad hoc file - getPath(adHocValue.womValue.value).toErrorOr.map(adHocValue -> _) - }) + adHocValues + .traverse[ErrorOr, (AdHocValue, Path)] { adHocValue => + // Build an actionable Path from the ad hoc file + getPath(adHocValue.womValue.value).toErrorOr.map(adHocValue -> _) + } // Localize the values if necessary - .map(_.traverse[Future, LocalizedAdHocValue](localize.tupled)).toEither + .map(_.traverse[Future, LocalizedAdHocValue](localize.tupled)) + .toEither // TODO: Asynchronify // This is obviously sad but turning it into a Future has earth-shattering consequences, so synchronizing it for now .flatMap(future => Try(Await.result(future, 1.hour)).toChecked) @@ -556,37 +602,41 @@ trait StandardAsyncExecutionActor val callable = jobDescriptor.taskCall.callable /* - * Try to map the command line values. - * - * May not work as the commandLineValueMapper was originally meant to modify paths in the later stages of command - * line instantiation. However, due to [[AdHocValue]] support the command line instantiation itself currently needs - * to use the commandLineValue mapper. So the commandLineValueMapper is attempted first, and if that fails then - * returns the original womValue. - */ - def tryCommandLineValueMapper(womValue: WomValue): WomValue = { + * Try to map the command line values. + * + * May not work as the commandLineValueMapper was originally meant to modify paths in the later stages of command + * line instantiation. However, due to [[AdHocValue]] support the command line instantiation itself currently needs + * to use the commandLineValue mapper. So the commandLineValueMapper is attempted first, and if that fails then + * returns the original womValue. + */ + def tryCommandLineValueMapper(womValue: WomValue): WomValue = Try(commandLineJobInputValueMapper(womValue)).getOrElse(womValue) - } - val unmappedInputs: Map[String, WomValue] = jobDescriptor.evaluatedTaskInputs.map({ + val unmappedInputs: Map[String, WomValue] = jobDescriptor.evaluatedTaskInputs.map { case (inputDefinition, womValue) => inputDefinition.localName.value -> womValue - }) - - val mappedInputs: Checked[Map[String, WomValue]] = localizedInputs.toErrorOr.map( - _.map({ - case (inputDefinition, value) => inputDefinition.localName.value -> tryCommandLineValueMapper(value) - }) - ).toEither - - val evaluateAndInitialize = (containerizedInputExpression: ContainerizedInputExpression) => for { - mapped <- mappedInputs - evaluated <- containerizedInputExpression.evaluate(unmappedInputs, mapped, backendEngineFunctions).toChecked - initialized <- evaluated.traverse[IOChecked, AdHocValue]({ adHocValue => - adHocValue.womValue.initialize(backendEngineFunctions).map({ - case file: WomFile => adHocValue.copy(womValue = file) - case _ => adHocValue - }) - }).toChecked - } yield initialized + } + + val mappedInputs: Checked[Map[String, WomValue]] = localizedInputs.toErrorOr + .map( + _.map { case (inputDefinition, value) => + inputDefinition.localName.value -> tryCommandLineValueMapper(value) + } + ) + .toEither + + val evaluateAndInitialize = (containerizedInputExpression: ContainerizedInputExpression) => + for { + mapped <- mappedInputs + evaluated <- containerizedInputExpression.evaluate(unmappedInputs, mapped, backendEngineFunctions).toChecked + initialized <- evaluated + .traverse[IOChecked, AdHocValue] { adHocValue => + adHocValue.womValue.initialize(backendEngineFunctions).map { + case file: WomFile => adHocValue.copy(womValue = file) + case _ => adHocValue + } + } + .toChecked + } yield initialized callable.adHocFileCreation.toList .flatTraverse[ErrorOr, AdHocValue](evaluateAndInitialize.andThen(_.toValidated)) @@ -596,9 +646,10 @@ trait StandardAsyncExecutionActor .flatMap(localizeAdHocValues.andThen(_.toEither)) .toValidated - protected def asAdHocFile(womFile: WomFile): Option[AdHocValue] = evaluatedAdHocFiles map { _.find({ - case AdHocValue(file, _, _) => file.value == womFile.value - }) + protected def asAdHocFile(womFile: WomFile): Option[AdHocValue] = evaluatedAdHocFiles map { + _.find { case AdHocValue(file, _, _) => + file.value == womFile.value + } } getOrElse None protected def isAdHocFile(womFile: WomFile): Boolean = asAdHocFile(womFile).isDefined @@ -608,18 +659,22 @@ trait StandardAsyncExecutionActor val callable = jobDescriptor.taskCall.callable // Replace input files with the ad hoc updated version - def adHocFilePreProcessor(in: WomEvaluatedCallInputs): Try[WomEvaluatedCallInputs] = { + def adHocFilePreProcessor(in: WomEvaluatedCallInputs): Try[WomEvaluatedCallInputs] = localizedAdHocValues.toTry("Error evaluating ad hoc files") map { adHocFiles => - in map { - case (inputDefinition, originalWomValue) => - inputDefinition -> adHocFiles.collectFirst({ - case AsAdHocValue(AdHocValue(originalWomFile, _, Some(inputName))) if inputName == inputDefinition.localName.value => originalWomFile - case AsLocalizedAdHocValue(LocalizedAdHocValue(AdHocValue(originalWomFile, _, Some(inputName)), localizedPath)) if inputName == inputDefinition.localName.value => + in map { case (inputDefinition, originalWomValue) => + inputDefinition -> adHocFiles + .collectFirst { + case AsAdHocValue(AdHocValue(originalWomFile, _, Some(inputName))) + if inputName == inputDefinition.localName.value => + originalWomFile + case AsLocalizedAdHocValue( + LocalizedAdHocValue(AdHocValue(originalWomFile, _, Some(inputName)), localizedPath) + ) if inputName == inputDefinition.localName.value => originalWomFile.mapFile(_ => localizedPath.pathAsString) - }).getOrElse(originalWomValue) + } + .getOrElse(originalWomValue) } } - } // Gets the inputs that will be mutated by instantiating the command. val mutatingPreProcessor: WomEvaluatedCallInputs => Try[WomEvaluatedCallInputs] = { _ => @@ -633,16 +688,19 @@ trait StandardAsyncExecutionActor jobDescriptor, backendEngineFunctions, mutatingPreProcessor, - commandLineValueMapper, - runtimeEnvironment + commandLineValueMapper ) - def makeStringKeyedMap(list: List[(LocalName, WomValue)]): Map[String, WomValue] = list.toMap map { case (k, v) => k.value -> v } + def makeStringKeyedMap(list: List[(LocalName, WomValue)]): Map[String, WomValue] = list.toMap map { case (k, v) => + k.value -> v + } val command = instantiatedCommandValidation flatMap { instantiatedCommand => val valueMappedPreprocessedInputs = instantiatedCommand.valueMappedPreprocessedInputs |> makeStringKeyedMap - val adHocFileCreationSideEffectFiles: ErrorOr[List[CommandSetupSideEffectFile]] = localizedAdHocValues map { _.map(adHocValueToCommandSetupSideEffectFile) } + val adHocFileCreationSideEffectFiles: ErrorOr[List[CommandSetupSideEffectFile]] = localizedAdHocValues map { + _.map(adHocValueToCommandSetupSideEffectFile) + } def evaluateEnvironmentExpression(nameAndExpression: (String, WomExpression)): ErrorOr[(String, String)] = { val (name, expression) = nameAndExpression @@ -653,16 +711,25 @@ trait StandardAsyncExecutionActor // Build a list of functions from a CommandTaskDefinition to an Option[WomExpression] representing a possible // redirection or override of the filename of a redirection. Evaluate that expression if present and stringify. - val List(stdinRedirect, stdoutOverride, stderrOverride) = List[CommandTaskDefinition => Option[WomExpression]]( - _.stdinRedirection, _.stdoutOverride, _.stderrOverride) map { - _.apply(callable).traverse[ErrorOr, String] { _.evaluateValue(valueMappedPreprocessedInputs, backendEngineFunctions) map { _.valueString} } - } + val List(stdinRedirect, stdoutOverride, stderrOverride) = + List[CommandTaskDefinition => Option[WomExpression]](_.stdinRedirection, + _.stdoutOverride, + _.stderrOverride + ) map { + _.apply(callable).traverse[ErrorOr, String] { + _.evaluateValue(valueMappedPreprocessedInputs, backendEngineFunctions) map { _.valueString } + } + } (adHocFileCreationSideEffectFiles, environmentVariables, stdinRedirect, stdoutOverride, stderrOverride) mapN { (adHocFiles, env, in, out, err) => instantiatedCommand.copy( - createdFiles = instantiatedCommand.createdFiles ++ adHocFiles, environmentVariables = env.toMap, - evaluatedStdinRedirection = in, evaluatedStdoutOverride = out, evaluatedStderrOverride = err) + createdFiles = instantiatedCommand.createdFiles ++ adHocFiles, + environmentVariables = env.toMap, + evaluatedStdinRedirection = in, + evaluatedStdoutOverride = out, + evaluatedStderrOverride = err + ) }: ErrorOr[InstantiatedCommand] } @@ -679,11 +746,10 @@ trait StandardAsyncExecutionActor * * If the `command` errors for some reason, put a "-1" into the rc file. */ - def redirectOutputs(command: String): String = { + def redirectOutputs(command: String): String = // > 128 is the cutoff for signal-induced process deaths such as might be observed with abort. // http://www.tldp.org/LDP/abs/html/exitcodes.html s"""$command < /dev/null || { rc=$$?; if [ "$$rc" -gt "128" ]; then echo $$rc; else echo -1; fi } > ${jobPaths.returnCode}""" - } /** A tag that may be used for logging. */ lazy val tag = s"${this.getClass.getSimpleName} [UUID(${workflowIdForLogging.shortString}):${jobDescriptor.key.tag}]" @@ -693,42 +759,47 @@ trait StandardAsyncExecutionActor * * @return True if a non-empty `remoteStdErrPath` should fail the job. */ - lazy val failOnStdErr: Boolean = RuntimeAttributesValidation.extract( - FailOnStderrValidation.instance, validatedRuntimeAttributes) + lazy val failOnStdErr: Boolean = + RuntimeAttributesValidation.extract(FailOnStderrValidation.instance, validatedRuntimeAttributes) /** - * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. - * - * @return the behavior for continuing on the return code. - */ - lazy val continueOnReturnCode: ContinueOnReturnCode = RuntimeAttributesValidation.extract( - ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) + * Returns the behavior for continuing on the return code, obtained by converting `returnCodeContents` to an Int. + * + * @return the behavior for continuing on the return code. + */ + lazy val continueOnReturnCode: ContinueOnReturnCode = + RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) /** * Returns the max number of times that a failed job should be retried, obtained by converting `maxRetries` to an Int. */ - lazy val maxRetries: Int = RuntimeAttributesValidation.extract( - MaxRetriesValidation.instance, validatedRuntimeAttributes) + lazy val maxRetries: Int = + RuntimeAttributesValidation.extract(MaxRetriesValidation.instance, validatedRuntimeAttributes) - - lazy val previousFailedRetries: Int = jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.FailedRetryCountKey) match { - case Some(KvPair(_,v)) => v.toInt - case _ => 0 - } + lazy val previousFailedRetries: Int = + jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.FailedRetryCountKey) match { + case Some(KvPair(_, v)) => v.toInt + case _ => 0 + } /** * Returns the memory multiplier for previous attempt if available */ - lazy val previousMemoryMultiplier: Option[Double] = jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.MemoryMultiplierKey) match { - case Some(KvPair(_,v)) => Try(v.toDouble) match { - case Success(m) => Option(m) - case Failure(e) => - // should not happen as Cromwell itself had written the value as a Double - log.error(e, s"Programmer error: unexpected failure attempting to convert value of MemoryMultiplierKey from JOB_KEY_VALUE_ENTRY table to Double.") - None + lazy val previousMemoryMultiplier: Option[Double] = + jobDescriptor.prefetchedKvStoreEntries.get(BackendLifecycleActorFactory.MemoryMultiplierKey) match { + case Some(KvPair(_, v)) => + Try(v.toDouble) match { + case Success(m) => Option(m) + case Failure(e) => + // should not happen as Cromwell itself had written the value as a Double + log.error( + e, + s"Programmer error: unexpected failure attempting to convert value of MemoryMultiplierKey from JOB_KEY_VALUE_ENTRY table to Double." + ) + None + } + case _ => None } - case _ => None - } /** * Execute the job specified in the params. Should return a `StandardAsyncPendingExecutionHandle`, or a @@ -736,9 +807,8 @@ trait StandardAsyncExecutionActor * * @return the execution handle for the job. */ - def execute(): ExecutionHandle = { + def execute(): ExecutionHandle = throw new UnsupportedOperationException(s"Neither execute() nor executeAsync() implemented by $getClass") - } /** * Async execute the job specified in the params. Should return a `StandardAsyncPendingExecutionHandle`, or a @@ -776,7 +846,8 @@ trait StandardAsyncExecutionActor * @param jobId The previously recorded job id. * @return the execution handle for the job. */ - def reconnectToAbortAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) + def reconnectToAbortAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = + Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) /** * This is in spirit similar to recover except it does not defaults back to running the job if not implemented. @@ -788,7 +859,8 @@ trait StandardAsyncExecutionActor * @param jobId The previously recorded job id. * @return the execution handle for the job. */ - def reconnectAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) + def reconnectAsync(jobId: StandardAsyncJob): Future[ExecutionHandle] = + Future.failed(JobReconnectionNotSupportedException(jobDescriptor.key)) /** * Returns the run status for the job. @@ -796,9 +868,8 @@ trait StandardAsyncExecutionActor * @param handle The handle of the running job. * @return The status of the job. */ - def pollStatus(handle: StandardAsyncPendingExecutionHandle): StandardAsyncRunState = { + def pollStatus(handle: StandardAsyncPendingExecutionHandle): StandardAsyncRunState = throw new UnsupportedOperationException(s"Neither pollStatus nor pollStatusAsync implemented by $getClass") - } /** * Returns the async run status for the job. @@ -806,7 +877,8 @@ trait StandardAsyncExecutionActor * @param handle The handle of the running job. * @return The status of the job. */ - def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[StandardAsyncRunState] = Future.fromTry(Try(pollStatus(handle))) + def pollStatusAsync(handle: StandardAsyncPendingExecutionHandle): Future[StandardAsyncRunState] = + Future.fromTry(Try(pollStatus(handle))) /** * Adds custom behavior invoked when polling fails due to some exception. By default adds nothing. @@ -817,9 +889,8 @@ trait StandardAsyncExecutionActor * * @return A partial function handler for exceptions after polling. */ - def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = { + def customPollStatusFailure: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = PartialFunction.empty - } /** * Returns true when a job is complete, either successfully or unsuccessfully. @@ -890,13 +961,12 @@ trait StandardAsyncExecutionActor * * By default handles the behavior of `requestsAbortAndDiesImmediately`. */ - def postAbort(): Unit = { + def postAbort(): Unit = if (requestsAbortAndDiesImmediately) { tellMetadata(Map(CallMetadataKeys.BackendStatus -> "Aborted")) context.parent ! JobAbortedResponse(jobDescriptor.key) context.stop(self) } - } /** * Output value mapper. @@ -904,19 +974,17 @@ trait StandardAsyncExecutionActor * @param womValue The original WOM value. * @return The Try wrapped and mapped WOM value. */ - final def outputValueMapper(womValue: WomValue): Try[WomValue] = { + final def outputValueMapper(womValue: WomValue): Try[WomValue] = WomFileMapper.mapWomFiles(mapOutputWomFile)(womValue) - } /** * Used to convert to output paths. * */ def mapOutputWomFile(womFile: WomFile): WomFile = - womFile.mapFile{ - path => - val pathFromContainerInputs = jobPaths.hostPathFromContainerInputs(path) - pathFromContainerInputs.toAbsolutePath.toString + womFile.mapFile { path => + val pathFromContainerInputs = jobPaths.hostPathFromContainerInputs(path) + pathFromContainerInputs.toAbsolutePath.toString } /** @@ -926,9 +994,8 @@ trait StandardAsyncExecutionActor * * @return A Try wrapping evaluated outputs. */ - def evaluateOutputs()(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = { + def evaluateOutputs()(implicit ec: ExecutionContext): Future[EvaluatedJobOutputs] = OutputEvaluator.evaluateOutputs(jobDescriptor, backendEngineFunctions, outputValueMapper) - } /** * Tests whether an attempted result of evaluateOutputs should possibly be retried. @@ -944,7 +1011,7 @@ trait StandardAsyncExecutionActor * @param exception The exception, possibly an CromwellAggregatedException. * @return True if evaluateOutputs should be retried later. */ - final def retryEvaluateOutputsAggregated(exception: Exception): Boolean = { + final def retryEvaluateOutputsAggregated(exception: Exception): Boolean = exception match { case aggregated: CromwellAggregatedException => aggregated.throwables.collectFirst { @@ -952,7 +1019,6 @@ trait StandardAsyncExecutionActor }.isDefined case _ => retryEvaluateOutputs(exception) } - } /** * Tests whether an attempted result of evaluateOutputs should possibly be retried. @@ -978,7 +1044,8 @@ trait StandardAsyncExecutionActor */ def handleExecutionSuccess(runStatus: StandardAsyncRunState, handle: StandardAsyncPendingExecutionHandle, - returnCode: Int)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { + returnCode: Int + )(implicit ec: ExecutionContext): Future[ExecutionHandle] = evaluateOutputs() map { case ValidJobOutputs(outputs) => // Need to make sure the paths are up to date before sending the detritus back in the response @@ -995,7 +1062,6 @@ trait StandardAsyncExecutionActor handle case JobOutputsEvaluationException(ex) => FailedNonRetryableExecutionHandle(ex, kvPairsToSave = None) } - } /** * Process an unsuccessful run, as interpreted by `handleExecutionFailure`. @@ -1003,18 +1069,18 @@ trait StandardAsyncExecutionActor * @return The execution handle. */ def retryElseFail(backendExecutionStatus: Future[ExecutionHandle], - retryWithMoreMemory: Boolean = false): Future[ExecutionHandle] = { - + retryWithMoreMemory: Boolean = false + ): Future[ExecutionHandle] = backendExecutionStatus flatMap { case failedRetryableOrNonRetryable: FailedExecutionHandle => - val kvsFromPreviousAttempt = jobDescriptor.prefetchedKvStoreEntries.collect { case (key: String, kvPair: KvPair) => key -> kvPair } val kvsForNextAttempt = failedRetryableOrNonRetryable.kvPairsToSave match { - case Some(kvPairs) => kvPairs.map { - case kvPair@KvPair(ScopedKey(_, _, key), _) => key -> kvPair - }.toMap + case Some(kvPairs) => + kvPairs.map { case kvPair @ KvPair(ScopedKey(_, _, key), _) => + key -> kvPair + }.toMap case None => Map.empty[String, KvPair] } @@ -1024,23 +1090,35 @@ trait StandardAsyncExecutionActor (retryWithMoreMemory, memoryRetryFactor, previousMemoryMultiplier) match { case (true, Some(retryFactor), Some(previousMultiplier)) => val nextMemoryMultiplier = previousMultiplier * retryFactor.value - saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true, Option(nextMemoryMultiplier)) + saveAttrsAndRetry(failed, + kvsFromPreviousAttempt, + kvsForNextAttempt, + incFailedCount = true, + Option(nextMemoryMultiplier) + ) case (true, Some(retryFactor), None) => - saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true, Option(retryFactor.value)) - case (_, _, _) => saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true) + saveAttrsAndRetry(failed, + kvsFromPreviousAttempt, + kvsForNextAttempt, + incFailedCount = true, + Option(retryFactor.value) + ) + case (_, _, _) => + saveAttrsAndRetry(failed, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = true) } case failedNonRetryable: FailedNonRetryableExecutionHandle => Future.successful(failedNonRetryable) - case failedRetryable: FailedRetryableExecutionHandle => saveAttrsAndRetry(failedRetryable, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = false) + case failedRetryable: FailedRetryableExecutionHandle => + saveAttrsAndRetry(failedRetryable, kvsFromPreviousAttempt, kvsForNextAttempt, incFailedCount = false) } case _ => backendExecutionStatus } - } private def saveAttrsAndRetry(failedExecHandle: FailedExecutionHandle, kvPrev: Map[String, KvPair], kvNext: Map[String, KvPair], incFailedCount: Boolean, - nextMemoryMultiplier: Option[Double] = None): Future[FailedRetryableExecutionHandle] = { + nextMemoryMultiplier: Option[Double] = None + ): Future[FailedRetryableExecutionHandle] = failedExecHandle match { case failedNonRetryable: FailedNonRetryableExecutionHandle => saveKvPairsForNextAttempt(kvPrev, kvNext, incFailedCount, nextMemoryMultiplier) map { _ => @@ -1049,7 +1127,6 @@ trait StandardAsyncExecutionActor case failedRetryable: FailedRetryableExecutionHandle => saveKvPairsForNextAttempt(kvPrev, kvNext, incFailedCount, nextMemoryMultiplier) map (_ => failedRetryable) } - } /** * Merge key-value pairs from previous job execution attempt with incoming pairs from current attempt, which has just @@ -1060,8 +1137,10 @@ trait StandardAsyncExecutionActor private def saveKvPairsForNextAttempt(kvsFromPreviousAttempt: Map[String, KvPair], kvsForNextAttempt: Map[String, KvPair], incrementFailedRetryCount: Boolean, - nextMemoryMultiplierOption: Option[Double]): Future[Seq[KvResponse]] = { - val nextKvJobKey = KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt + 1) + nextMemoryMultiplierOption: Option[Double] + ): Future[Seq[KvResponse]] = { + val nextKvJobKey = + KvJobKey(jobDescriptor.key.call.fullyQualifiedName, jobDescriptor.key.index, jobDescriptor.key.attempt + 1) def getNextKvPair[A](key: String, value: String): Map[String, KvPair] = { val nextScopedKey = ScopedKey(jobDescriptor.workflowDescriptor.id, nextKvJobKey, key) @@ -1069,7 +1148,8 @@ trait StandardAsyncExecutionActor Map(key -> nextKvPair) } - val kvsFromPreviousAttemptUpd = kvsFromPreviousAttempt.view.mapValues(kvPair => kvPair.copy(key = kvPair.key.copy(jobKey = nextKvJobKey))) + val kvsFromPreviousAttemptUpd = + kvsFromPreviousAttempt.view.mapValues(kvPair => kvPair.copy(key = kvPair.key.copy(jobKey = nextKvJobKey))) val failedRetryCountKvPair: Map[String, KvPair] = if (incrementFailedRetryCount) getNextKvPair(FailedRetryCountKey, (previousFailedRetries + 1).toString) @@ -1087,8 +1167,10 @@ trait StandardAsyncExecutionActor if (failures.isEmpty) { respSeq } else { - throw new RuntimeException("Failed to save one or more job execution attributes to the database between " + - "attempts:\n " + failures.mkString("\n")) + throw new RuntimeException( + "Failed to save one or more job execution attributes to the database between " + + "attempts:\n " + failures.mkString("\n") + ) } } } @@ -1099,8 +1181,7 @@ trait StandardAsyncExecutionActor * @param runStatus The run status. * @return The execution handle. */ - def handleExecutionFailure(runStatus: StandardAsyncRunState, - returnCode: Option[Int]): Future[ExecutionHandle] = { + def handleExecutionFailure(runStatus: StandardAsyncRunState, returnCode: Option[Int]): Future[ExecutionHandle] = { val exception = new RuntimeException(s"Task ${jobDescriptor.key.tag} failed for unknown reason: $runStatus") Future.successful(FailedNonRetryableExecutionHandle(exception, returnCode, None)) } @@ -1129,67 +1210,67 @@ trait StandardAsyncExecutionActor } override def executeOrRecover(mode: ExecutionMode)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { - val executeOrRecoverFuture = { + val executeOrRecoverFuture = mode match { - case Reconnect(jobId: StandardAsyncJob@unchecked) => reconnectAsync(jobId) - case ReconnectToAbort(jobId: StandardAsyncJob@unchecked) => reconnectToAbortAsync(jobId) - case Recover(jobId: StandardAsyncJob@unchecked) => recoverAsync(jobId) + case Reconnect(jobId: StandardAsyncJob @unchecked) => reconnectAsync(jobId) + case ReconnectToAbort(jobId: StandardAsyncJob @unchecked) => reconnectToAbortAsync(jobId) + case Recover(jobId: StandardAsyncJob @unchecked) => recoverAsync(jobId) case _ => tellMetadata(startMetadataKeyValues) executeAsync() } - } - executeOrRecoverFuture flatMap executeOrRecoverSuccess recoverWith { - case throwable: Throwable => Future failed { + executeOrRecoverFuture flatMap executeOrRecoverSuccess recoverWith { case throwable: Throwable => + Future failed { jobLogger.error(s"Error attempting to $mode", throwable) throwable } } } - private def executeOrRecoverSuccess(executionHandle: ExecutionHandle): Future[ExecutionHandle] = { + private def executeOrRecoverSuccess(executionHandle: ExecutionHandle): Future[ExecutionHandle] = executionHandle match { - case handle: PendingExecutionHandle[StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunState@unchecked] => - - configurationDescriptor.slowJobWarningAfter foreach { duration => self ! WarnAboutSlownessAfter(handle.pendingJob.jobId, duration) } + case handle: PendingExecutionHandle[StandardAsyncJob @unchecked, + StandardAsyncRunInfo @unchecked, + StandardAsyncRunState @unchecked + ] => + configurationDescriptor.slowJobWarningAfter foreach { duration => + self ! WarnAboutSlownessAfter(handle.pendingJob.jobId, duration) + } tellKvJobId(handle.pendingJob) map { _ => - if (logJobIds) jobLogger.info(s"job id: ${handle.pendingJob.jobId}") + if (logJobIds) jobLogger.debug(s"job id: ${handle.pendingJob.jobId}") tellMetadata(Map(CallMetadataKeys.JobId -> handle.pendingJob.jobId)) /* NOTE: Because of the async nature of the Scala Futures, there is a point in time where we have submitted this or the prior runnable to the thread pool this actor doesn't know the job id for aborting. These runnables are queued up and may still be run by the thread pool anytime in the future. Issue #1218 may address this inconsistency at a later time. For now, just go back and check if we missed the abort command. - */ + */ self ! CheckMissedAbort(handle.pendingJob) executionHandle } case _ => Future.successful(executionHandle) } - } - override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = { + override def poll(previous: ExecutionHandle)(implicit ec: ExecutionContext): Future[ExecutionHandle] = previous match { - case handle: PendingExecutionHandle[ - StandardAsyncJob@unchecked, StandardAsyncRunInfo@unchecked, StandardAsyncRunState@unchecked] => - + case handle: PendingExecutionHandle[StandardAsyncJob @unchecked, + StandardAsyncRunInfo @unchecked, + StandardAsyncRunState @unchecked + ] => jobLogger.debug(s"$tag Polling Job ${handle.pendingJob}") - pollStatusAsync(handle) flatMap { - backendRunStatus => - self ! WarnAboutSlownessIfNecessary - handlePollSuccess(handle, backendRunStatus) - } recover { - case throwable => - handlePollFailure(handle, throwable) + pollStatusAsync(handle) flatMap { backendRunStatus => + self ! WarnAboutSlownessIfNecessary + handlePollSuccess(handle, backendRunStatus) + } recover { case throwable => + handlePollFailure(handle, throwable) } case successful: SuccessfulExecutionHandle => Future.successful(successful) case failed: FailedNonRetryableExecutionHandle => Future.successful(failed) case failedRetryable: FailedRetryableExecutionHandle => Future.successful(failedRetryable) case badHandle => Future.failed(new IllegalArgumentException(s"Unexpected execution handle: $badHandle")) } - } /** * Process a poll success. @@ -1199,7 +1280,8 @@ trait StandardAsyncExecutionActor * @return The updated execution handle. */ def handlePollSuccess(oldHandle: StandardAsyncPendingExecutionHandle, - state: StandardAsyncRunState): Future[ExecutionHandle] = { + state: StandardAsyncRunState + ): Future[ExecutionHandle] = { val previousState = oldHandle.previousState if (!(previousState exists statusEquivalentTo(state))) { // If this is the first time checking the status, we log the transition as '-' to 'currentStatus'. Otherwise just use @@ -1215,7 +1297,10 @@ trait StandardAsyncExecutionActor val metadata = getTerminalMetadata(state) tellMetadata(metadata) handleExecutionResult(state, oldHandle) - case s => Future.successful(oldHandle.copy(previousState = Option(s))) // Copy the current handle with updated previous status. + case s => + Future.successful( + oldHandle.copy(previousState = Option(s)) + ) // Copy the current handle with updated previous status. } } @@ -1226,8 +1311,7 @@ trait StandardAsyncExecutionActor * @param throwable The cause of the polling failure. * @return The updated execution handle. */ - def handlePollFailure(oldHandle: StandardAsyncPendingExecutionHandle, - throwable: Throwable): ExecutionHandle = { + def handlePollFailure(oldHandle: StandardAsyncPendingExecutionHandle, throwable: Throwable): ExecutionHandle = throwable match { case exception: Exception => val handler: PartialFunction[(ExecutionHandle, Exception), ExecutionHandle] = @@ -1238,7 +1322,9 @@ trait StandardAsyncExecutionActor FailedNonRetryableExecutionHandle(exception, kvPairsToSave = None) case (handle: ExecutionHandle, exception: Exception) => // Log exceptions and return the original handle to try again. - jobLogger.warn(s"Caught non-fatal ${exception.getClass.getSimpleName} exception trying to poll, retrying", exception) + jobLogger.warn(s"Caught non-fatal ${exception.getClass.getSimpleName} exception trying to poll, retrying", + exception + ) handle } handler((oldHandle, exception)) @@ -1247,7 +1333,6 @@ trait StandardAsyncExecutionActor // Someone has subclassed or instantiated Throwable directly. Kill the job. They should be using an Exception. FailedNonRetryableExecutionHandle(throwable, kvPairsToSave = None) } - } /** * Process an execution result. @@ -1257,33 +1342,39 @@ trait StandardAsyncExecutionActor * @return The updated execution handle. */ def handleExecutionResult(status: StandardAsyncRunState, - oldHandle: StandardAsyncPendingExecutionHandle): Future[ExecutionHandle] = { + oldHandle: StandardAsyncPendingExecutionHandle + ): Future[ExecutionHandle] = { + // Returns true if the task has written an RC file that indicates OOM, false otherwise def memoryRetryRC: Future[Boolean] = { + // convert int to boolean def returnCodeAsBoolean(codeAsOption: Option[String]): Boolean = { codeAsOption match { case Some(codeAsString) => Try(codeAsString.trim.toInt) match { - case Success(code) => code match { - case StderrContainsRetryKeysCode => true - case _ => false - } + case Success(code) => + code match { + case StderrContainsRetryKeysCode => true + case _ => false + } case Failure(e) => - log.error(s"'CheckingForMemoryRetry' action exited with code '$codeAsString' which couldn't be " + - s"converted to an Integer. Task will not be retried with more memory. Error: ${ExceptionUtils.getMessage(e)}") + log.error( + s"'CheckingForMemoryRetry' action exited with code '$codeAsString' which couldn't be " + + s"converted to an Integer. Task will not be retried with more memory. Error: ${ExceptionUtils.getMessage(e)}" + ) false } case None => false } } - + // read if the file exists def readMemoryRetryRCFile(fileExists: Boolean): Future[Option[String]] = { if (fileExists) asyncIo.contentAsStringAsync(jobPaths.memoryRetryRC, None, failOnOverflow = false).map(Option(_)) else Future.successful(None) } - + //finally : assign the yielded variable for { fileExists <- asyncIo.existsAsync(jobPaths.memoryRetryRC) retryCheckRCAsOption <- readMemoryRetryRCFile(fileExists) @@ -1291,58 +1382,109 @@ trait StandardAsyncExecutionActor } yield retryWithMoreMemory } + // get the exit code of the job. + def JobExitCode: Future[String] = { + + // read if the file exists + def readRCFile(fileExists: Boolean): Future[String] = { + if (fileExists) + asyncIo.contentAsStringAsync(jobPaths.returnCode, None, failOnOverflow = false) + else { + jobLogger.warn("RC file not found. Setting job to failed & waiting 5m before retry.") + Thread.sleep(300000) + Future("1") + } + } + //finally : assign the yielded variable + for { + fileExists <- asyncIo.existsAsync(jobPaths.returnCode) + jobRC <- readRCFile(fileExists) + } yield jobRC + } + + // get path to sderr val stderr = jobPaths.standardPaths.error lazy val stderrAsOption: Option[Path] = Option(stderr) - + // get the three needed variables, using functions above or direct assignment. val stderrSizeAndReturnCodeAndMemoryRetry = for { - returnCodeAsString <- asyncIo.contentAsStringAsync(jobPaths.returnCode, None, failOnOverflow = false) + returnCodeAsString <- JobExitCode // Only check stderr size if we need to, otherwise this results in a lot of unnecessary I/O that // may fail due to race conditions on quickly-executing jobs. stderrSize <- if (failOnStdErr) asyncIo.sizeAsync(stderr) else Future.successful(0L) - retryWithMoreMemory <- memoryRetryRC - } yield (stderrSize, returnCodeAsString, retryWithMoreMemory) - - stderrSizeAndReturnCodeAndMemoryRetry flatMap { - case (stderrSize, returnCodeAsString, retryWithMoreMemory) => - val tryReturnCodeAsInt = Try(returnCodeAsString.trim.toInt) - - if (isDone(status)) { - tryReturnCodeAsInt match { - case Success(returnCodeAsInt) if failOnStdErr && stderrSize.intValue > 0 => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, stderrSize, stderrAsOption), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle) - case Success(returnCodeAsInt) if isAbort(returnCodeAsInt) => - Future.successful(AbortedExecutionHandle) - case Success(returnCodeAsInt) if continueOnReturnCode.continueFor(returnCodeAsInt) => - handleExecutionSuccess(status, oldHandle, returnCodeAsInt) - case Success(returnCodeAsInt) if retryWithMoreMemory => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle, retryWithMoreMemory) - case Success(returnCodeAsInt) => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt, stderrAsOption), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle) - case Failure(_) => - Future.successful(FailedNonRetryableExecutionHandle(ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString, stderrAsOption), kvPairsToSave = None)) - } - } else { - tryReturnCodeAsInt match { - case Success(returnCodeAsInt) if retryWithMoreMemory && !continueOnReturnCode.continueFor(returnCodeAsInt) => - val executionHandle = Future.successful(FailedNonRetryableExecutionHandle(RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), Option(returnCodeAsInt), None)) - retryElseFail(executionHandle, retryWithMoreMemory) - case _ => - val failureStatus = handleExecutionFailure(status, tryReturnCodeAsInt.toOption) - retryElseFail(failureStatus) - } + outOfMemoryDetected <- memoryRetryRC + } yield (stderrSize, returnCodeAsString, outOfMemoryDetected) + + stderrSizeAndReturnCodeAndMemoryRetry flatMap { case (stderrSize, returnCodeAsString, outOfMemoryDetected) => + val tryReturnCodeAsInt = Try(returnCodeAsString.trim.toInt) + + if (isDone(status)) { + tryReturnCodeAsInt match { + case Success(returnCodeAsInt) if failOnStdErr && stderrSize.intValue > 0 => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle(StderrNonEmpty(jobDescriptor.key.tag, stderrSize, stderrAsOption), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle) + case Success(returnCodeAsInt) if continueOnReturnCode.continueFor(returnCodeAsInt) => + handleExecutionSuccess(status, oldHandle, returnCodeAsInt) + // It's important that we check retryWithMoreMemory case before isAbort. RC could be 137 in either case; + // if it was caused by OOM killer, want to handle as OOM and not job abort. + case Success(returnCodeAsInt) if outOfMemoryDetected && memoryRetryRequested => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle( + RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle, outOfMemoryDetected) + case Success(returnCodeAsInt) if isAbort(returnCodeAsInt) => + Future.successful(AbortedExecutionHandle) + case Success(returnCodeAsInt) => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle(WrongReturnCode(jobDescriptor.key.tag, returnCodeAsInt, stderrAsOption), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle) + case Failure(_) => + Future.successful( + FailedNonRetryableExecutionHandle( + ReturnCodeIsNotAnInt(jobDescriptor.key.tag, returnCodeAsString, stderrAsOption), + kvPairsToSave = None + ) + ) } - } recoverWith { - case exception => - if (isDone(status)) Future.successful(FailedNonRetryableExecutionHandle(exception, kvPairsToSave = None)) - else { - val failureStatus = handleExecutionFailure(status, None) - retryElseFail(failureStatus) + } else { + tryReturnCodeAsInt match { + case Success(returnCodeAsInt) + if outOfMemoryDetected && memoryRetryRequested && !continueOnReturnCode.continueFor(returnCodeAsInt) => + val executionHandle = Future.successful( + FailedNonRetryableExecutionHandle( + RetryWithMoreMemory(jobDescriptor.key.tag, stderrAsOption, memoryRetryErrorKeys, log), + Option(returnCodeAsInt), + None + ) + ) + retryElseFail(executionHandle, outOfMemoryDetected) + case _ => + val failureStatus = handleExecutionFailure(status, tryReturnCodeAsInt.toOption) + retryElseFail(failureStatus) } + } + } recoverWith { case exception => + if (isDone(status)) Future.successful(FailedNonRetryableExecutionHandle(exception, kvPairsToSave = None)) + else { + val failureStatus = handleExecutionFailure(status, None) + retryElseFail(failureStatus) + } } } + + /** * Send the job id of the running job to the key value store. @@ -1369,7 +1511,7 @@ trait StandardAsyncExecutionActor serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), metadataKeyValues) } - override protected implicit lazy val ec: ExecutionContextExecutor = context.dispatcher + implicit override protected lazy val ec: ExecutionContextExecutor = context.dispatcher } /** diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala index dd4254c8439..9d049ee8fff 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardCachingActorHelper.scala @@ -48,16 +48,16 @@ trait StandardCachingActorHelper extends JobCachingActorHelper { lazy val call: CommandCallNode = jobDescriptor.key.call - lazy val standardInitializationData: StandardInitializationData = BackendInitializationData. - as[StandardInitializationData](backendInitializationDataOption) + lazy val standardInitializationData: StandardInitializationData = + BackendInitializationData.as[StandardInitializationData](backendInitializationDataOption) lazy val validatedRuntimeAttributes: ValidatedRuntimeAttributes = { val builder = standardInitializationData.runtimeAttributesBuilder builder.build(jobDescriptor.runtimeAttributes, jobLogger) } - lazy val isDockerRun: Boolean = RuntimeAttributesValidation.extractOption( - DockerValidation.instance, validatedRuntimeAttributes).isDefined + lazy val isDockerRun: Boolean = + RuntimeAttributesValidation.extractOption(DockerValidation.instance, validatedRuntimeAttributes).isDefined /** * Returns the paths to the job. @@ -78,7 +78,7 @@ trait StandardCachingActorHelper extends JobCachingActorHelper { val fileMetadata = jobPaths.metadataPaths - runtimeAttributesMetadata ++ fileMetadata ++ nonStandardMetadata + nonStandardMetadata ++ runtimeAttributesMetadata ++ fileMetadata } /** diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala index 7782c2da901..ac4c39fc8d7 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardExpressionFunctions.scala @@ -24,11 +24,15 @@ case class DefaultStandardExpressionFunctionsParams(override val pathBuilders: P override val callContext: CallContext, override val ioActorProxy: ActorRef, override val executionContext: ExecutionContext - ) extends StandardExpressionFunctionsParams +) extends StandardExpressionFunctionsParams // TODO: Once we figure out premapping and postmapping, maybe we can standardize that behavior. Currently that's the most important feature that subclasses override. class StandardExpressionFunctions(val standardParams: StandardExpressionFunctionsParams) - extends GlobFunctions with DirectoryFunctions with ReadLikeFunctions with WriteFunctions with CallCorePathFunctions { + extends GlobFunctions + with DirectoryFunctions + with ReadLikeFunctions + with WriteFunctions + with CallCorePathFunctions { override lazy val ec = standardParams.executionContext @@ -41,6 +45,6 @@ class StandardExpressionFunctions(val standardParams: StandardExpressionFunction val callContext: CallContext = standardParams.callContext val writeDirectory: Path = callContext.root - + val isDocker: Boolean = callContext.isDocker } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala index 22eb6763b53..4bac258312e 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardFinalizationActor.scala @@ -24,8 +24,7 @@ trait StandardFinalizationActorParams { def configurationDescriptor: BackendConfigurationDescriptor } -case class DefaultStandardFinalizationActorParams -( +case class DefaultStandardFinalizationActorParams( workflowDescriptor: BackendWorkflowDescriptor, calls: Set[CommandCallNode], jobExecutionMap: JobExecutionMap, @@ -42,7 +41,7 @@ case class DefaultStandardFinalizationActorParams * @param standardParams Standard parameters. */ class StandardFinalizationActor(val standardParams: StandardFinalizationActorParams) - extends BackendWorkflowFinalizationActor { + extends BackendWorkflowFinalizationActor { override lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor override lazy val calls: Set[CommandCallNode] = standardParams.calls @@ -59,7 +58,7 @@ class StandardFinalizationActor(val standardParams: StandardFinalizationActorPar override def afterAll(): Future[Unit] = copyCallLogs() - lazy val logPaths: Seq[Path] = { + lazy val logPaths: Seq[Path] = for { actualWorkflowPath <- workflowPaths.toSeq (backendWorkflowDescriptor, keys) <- jobExecutionMap.toSeq @@ -67,9 +66,8 @@ class StandardFinalizationActor(val standardParams: StandardFinalizationActorPar jobPaths = actualWorkflowPath.toJobPaths(key, backendWorkflowDescriptor) logPath <- jobPaths.logPaths.values } yield logPath - } - protected def copyCallLogs(): Future[Unit] = { + protected def copyCallLogs(): Future[Unit] = /* NOTE: Only using one thread pool slot here to upload all the files for all the calls. Using the io-dispatcher defined in application.conf because this might take a while. @@ -77,21 +75,18 @@ class StandardFinalizationActor(val standardParams: StandardFinalizationActorPar pool for parallel uploads. Measure and optimize as necessary. Will likely need retry code at some level as well. - */ + */ workflowPaths match { case Some(paths) => Future(paths.finalCallLogsPath foreach copyCallLogs)(ioExecutionContext) case _ => Future.successful(()) } - } - private def copyCallLogs(callLogsPath: Path): Unit = { + private def copyCallLogs(callLogsPath: Path): Unit = copyLogs(callLogsPath, logPaths) - } - private def copyLogs(callLogsDirPath: Path, logPaths: Seq[Path]): Unit = { + private def copyLogs(callLogsDirPath: Path, logPaths: Seq[Path]): Unit = workflowPaths match { case Some(paths) => logPaths.foreach(PathCopier.copy(paths.executionRoot, _, callLogsDirPath)) case None => } - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala index 95e898d6711..c4adba3cd59 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationActor.scala @@ -4,7 +4,12 @@ import akka.actor.ActorRef import cromwell.backend.io.WorkflowPaths import cromwell.backend.validation.RuntimeAttributesDefault import cromwell.backend.wfs.WorkflowPathBuilder -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendWorkflowDescriptor, BackendWorkflowInitializationActor} +import cromwell.backend.{ + BackendConfigurationDescriptor, + BackendInitializationData, + BackendWorkflowDescriptor, + BackendWorkflowInitializationActor +} import cromwell.core.WorkflowOptions import cromwell.core.path.PathBuilder import wom.expression.WomExpression @@ -24,8 +29,7 @@ trait StandardInitializationActorParams { def configurationDescriptor: BackendConfigurationDescriptor } -case class DefaultInitializationActorParams -( +case class DefaultInitializationActorParams( workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], @@ -42,7 +46,7 @@ case class DefaultInitializationActorParams * @param standardParams Standard parameters */ class StandardInitializationActor(val standardParams: StandardInitializationActorParams) - extends BackendWorkflowInitializationActor { + extends BackendWorkflowInitializationActor { implicit protected val system = context.system @@ -50,16 +54,18 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo override lazy val calls: Set[CommandCallNode] = standardParams.calls - override def beforeAll(): Future[Option[BackendInitializationData]] = { + override def beforeAll(): Future[Option[BackendInitializationData]] = initializationData map Option.apply - } lazy val initializationData: Future[StandardInitializationData] = - workflowPaths map { new StandardInitializationData(_, runtimeAttributesBuilder, classOf[StandardExpressionFunctions]) } + workflowPaths map { + new StandardInitializationData(_, runtimeAttributesBuilder, classOf[StandardExpressionFunctions]) + } lazy val expressionFunctions: Class[_ <: StandardExpressionFunctions] = classOf[StandardExpressionFunctions] - lazy val pathBuilders: Future[List[PathBuilder]] = standardParams.configurationDescriptor.pathBuilders(workflowDescriptor.workflowOptions) + lazy val pathBuilders: Future[List[PathBuilder]] = + standardParams.configurationDescriptor.pathBuilders(workflowDescriptor.workflowOptions) lazy val workflowPaths: Future[WorkflowPaths] = pathBuilders map { WorkflowPathBuilder.workflowPaths(configurationDescriptor, workflowDescriptor, _) } @@ -74,13 +80,11 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo def runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder = StandardValidatedRuntimeAttributesBuilder.default(configurationDescriptor.backendRuntimeAttributesConfig) - override protected lazy val runtimeAttributeValidators: Map[String, (Option[WomExpression]) => Boolean] = { + override protected lazy val runtimeAttributeValidators: Map[String, (Option[WomExpression]) => Boolean] = runtimeAttributesBuilder.validatorMap - } - override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] = { + override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] = RuntimeAttributesDefault.workflowOptionsDefault(options, runtimeAttributesBuilder.coercionMap) - } def validateWorkflowOptions(): Try[Unit] = Success(()) @@ -93,19 +97,19 @@ class StandardInitializationActor(val standardParams: StandardInitializationActo val notSupportedAttrString = notSupportedAttributes mkString ", " workflowLogger.warn( s"Key/s [$notSupportedAttrString] is/are not supported by backend. " + - s"Unsupported attributes will not be part of job executions.") + s"Unsupported attributes will not be part of job executions." + ) } } } - override def validate(): Future[Unit] = { + override def validate(): Future[Unit] = Future.fromTry( for { _ <- validateWorkflowOptions() _ <- checkForUnsupportedRuntimeAttributes() } yield () ) - } override protected lazy val workflowDescriptor: BackendWorkflowDescriptor = standardParams.workflowDescriptor diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala index e2618818e50..d317af2ada2 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardInitializationData.scala @@ -6,8 +6,7 @@ import cromwell.backend.io.{JobPaths, WorkflowPaths} import scala.concurrent.ExecutionContext -class StandardInitializationData -( +class StandardInitializationData( val workflowPaths: WorkflowPaths, val runtimeAttributesBuilder: StandardValidatedRuntimeAttributesBuilder, val standardExpressionFunctionsClass: Class[_ <: StandardExpressionFunctions] @@ -17,7 +16,10 @@ class StandardInitializationData private lazy val standardExpressionFunctionsConstructor = standardExpressionFunctionsClass.getConstructor(classOf[StandardExpressionFunctionsParams]) - def expressionFunctions(jobPaths: JobPaths, ioActorProxy: ActorRef, ec: ExecutionContext): StandardExpressionFunctions = { + def expressionFunctions(jobPaths: JobPaths, + ioActorProxy: ActorRef, + ec: ExecutionContext + ): StandardExpressionFunctions = { val pathBuilders = jobPaths.workflowPaths.pathBuilders val standardParams = DefaultStandardExpressionFunctionsParams(pathBuilders, jobPaths.callContext, ioActorProxy, ec) standardExpressionFunctionsConstructor.newInstance(standardParams) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala index ac670617bb9..a595f589647 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardJobExecutionActorParams.scala @@ -1,12 +1,18 @@ package cromwell.backend.standard import akka.actor.ActorRef -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, MinimumRuntimeSettings} +import cromwell.backend.{ + BackendConfigurationDescriptor, + BackendInitializationData, + BackendJobDescriptor, + MinimumRuntimeSettings +} /** * Base trait for params passed to both the sync and async backend actors. */ trait StandardJobExecutionActorParams { + /** The service registry actor for key/value and metadata. */ def serviceRegistryActor: ActorRef diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala index 64f79f5ad7d..aec03977ad9 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardLifecycleActorFactory.scala @@ -16,6 +16,7 @@ import scala.concurrent.ExecutionContext * May be extended for using the standard sync/async backend pattern. */ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { + /** * Config values for the backend, and a pointer to the global config. * @@ -59,41 +60,65 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { * * @return the cache hit copying class. */ - lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = Option(classOf[DefaultStandardCacheHitCopyingActor]) + lazy val cacheHitCopyingActorClassOption: Option[Class[_ <: StandardCacheHitCopyingActor]] = Option( + classOf[DefaultStandardCacheHitCopyingActor] + ) /** * Returns the cache hit copying class. * * @return the cache hit copying class. */ - lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = Option(classOf[DefaultStandardFileHashingActor]) + lazy val fileHashingActorClassOption: Option[Class[_ <: StandardFileHashingActor]] = Option( + classOf[DefaultStandardFileHashingActor] + ) /** * Returns the finalization class. * * @return the finalization class. */ - lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] = Option(classOf[StandardFinalizationActor]) + lazy val finalizationActorClassOption: Option[Class[_ <: StandardFinalizationActor]] = Option( + classOf[StandardFinalizationActor] + ) - override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - serviceRegistryActor: ActorRef, restart: Boolean): Option[Props] = { + override def workflowInitializationActorProps(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + serviceRegistryActor: ActorRef, + restart: Boolean + ): Option[Props] = { val params = workflowInitializationActorParams(workflowDescriptor, ioActor, calls, serviceRegistryActor, restart) val props = Props(initializationActorClass, params).withDispatcher(Dispatcher.BackendDispatcher) Option(props) } - def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - serviceRegistryActor: ActorRef, restarting: Boolean): StandardInitializationActorParams = { - DefaultInitializationActorParams(workflowDescriptor, ioActor, calls, serviceRegistryActor, configurationDescriptor, restarting) - } + def workflowInitializationActorParams(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + serviceRegistryActor: ActorRef, + restarting: Boolean + ): StandardInitializationActorParams = + DefaultInitializationActorParams(workflowDescriptor, + ioActor, + calls, + serviceRegistryActor, + configurationDescriptor, + restarting + ) override def jobExecutionActorProps(jobDescriptor: BackendJobDescriptor, initializationDataOption: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActorOption: Option[ActorRef]): Props = { - val params = jobExecutionActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, - ioActor, backendSingletonActorOption) + backendSingletonActorOption: Option[ActorRef] + ): Props = { + val params = jobExecutionActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + backendSingletonActorOption + ) Props(new StandardSyncExecutionActor(params)).withDispatcher(Dispatcher.BackendDispatcher) } @@ -101,25 +126,35 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { initializationDataOption: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - backendSingletonActorOption: Option[ActorRef]): StandardSyncExecutionActorParams = { - DefaultStandardSyncExecutionActorParams(jobIdKey, serviceRegistryActor, ioActor, jobDescriptor, configurationDescriptor, - initializationDataOption, backendSingletonActorOption, asyncExecutionActorClass, MinimumRuntimeSettings()) - } + backendSingletonActorOption: Option[ActorRef] + ): StandardSyncExecutionActorParams = + DefaultStandardSyncExecutionActorParams( + jobIdKey, + serviceRegistryActor, + ioActor, + jobDescriptor, + configurationDescriptor, + initializationDataOption, + backendSingletonActorOption, + asyncExecutionActorClass, + MinimumRuntimeSettings() + ) - override def fileHashingActorProps: - Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = { - fileHashingActorClassOption map { - standardFileHashingActor => fileHashingActorInner(standardFileHashingActor) _ + override def fileHashingActorProps + : Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Option[ActorRef]) => Props] = + fileHashingActorClassOption map { standardFileHashingActor => + fileHashingActorInner(standardFileHashingActor) _ } - } - - def fileHashingActorInner(standardFileHashingActor: Class[_ <: StandardFileHashingActor]) - (jobDescriptor: BackendJobDescriptor, - initializationDataOption: Option[BackendInitializationData], - serviceRegistryActor: ActorRef, - ioActor: ActorRef, - fileHashCacheActor: Option[ActorRef]): Props = { - val params = fileHashingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, fileHashCacheActor) + + def fileHashingActorInner(standardFileHashingActor: Class[_ <: StandardFileHashingActor])( + jobDescriptor: BackendJobDescriptor, + initializationDataOption: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + ioActor: ActorRef, + fileHashCacheActor: Option[ActorRef] + ): Props = { + val params = + fileHashingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, fileHashCacheActor) Props(standardFileHashingActor, params).withDispatcher(BackendDispatcher) } @@ -127,26 +162,38 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { initializationDataOption: Option[BackendInitializationData], serviceRegistryActor: ActorRef, ioActor: ActorRef, - fileHashCacheActor: Option[ActorRef]): StandardFileHashingActorParams = { - DefaultStandardFileHashingActorParams( - jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, configurationDescriptor, fileHashCacheActor) - } + fileHashCacheActor: Option[ActorRef] + ): StandardFileHashingActorParams = + DefaultStandardFileHashingActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + configurationDescriptor, + fileHashCacheActor + ) - override def cacheHitCopyingActorProps: - Option[(BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props] = { - cacheHitCopyingActorClassOption map { - standardCacheHitCopyingActor => cacheHitCopyingActorInner(standardCacheHitCopyingActor) _ + override def cacheHitCopyingActorProps: Option[ + (BackendJobDescriptor, Option[BackendInitializationData], ActorRef, ActorRef, Int, Option[BlacklistCache]) => Props + ] = + cacheHitCopyingActorClassOption map { standardCacheHitCopyingActor => + cacheHitCopyingActorInner(standardCacheHitCopyingActor) _ } - } - def cacheHitCopyingActorInner(standardCacheHitCopyingActor: Class[_ <: StandardCacheHitCopyingActor]) - (jobDescriptor: BackendJobDescriptor, - initializationDataOption: Option[BackendInitializationData], - serviceRegistryActor: ActorRef, - ioActor: ActorRef, - cacheCopyAttempt: Int, - blacklistCache: Option[BlacklistCache]): Props = { - val params = cacheHitCopyingActorParams(jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, cacheCopyAttempt, blacklistCache) + def cacheHitCopyingActorInner(standardCacheHitCopyingActor: Class[_ <: StandardCacheHitCopyingActor])( + jobDescriptor: BackendJobDescriptor, + initializationDataOption: Option[BackendInitializationData], + serviceRegistryActor: ActorRef, + ioActor: ActorRef, + cacheCopyAttempt: Int, + blacklistCache: Option[BlacklistCache] + ): Props = { + val params = cacheHitCopyingActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + cacheCopyAttempt, + blacklistCache + ) Props(standardCacheHitCopyingActor, params).withDispatcher(BackendDispatcher) } @@ -155,71 +202,94 @@ trait StandardLifecycleActorFactory extends BackendLifecycleActorFactory { serviceRegistryActor: ActorRef, ioActor: ActorRef, cacheCopyAttempt: Int, - blacklistCache: Option[BlacklistCache]): StandardCacheHitCopyingActorParams = { - DefaultStandardCacheHitCopyingActorParams( - jobDescriptor, initializationDataOption, serviceRegistryActor, ioActor, configurationDescriptor, cacheCopyAttempt, blacklistCache) - } + blacklistCache: Option[BlacklistCache] + ): StandardCacheHitCopyingActorParams = + DefaultStandardCacheHitCopyingActorParams(jobDescriptor, + initializationDataOption, + serviceRegistryActor, + ioActor, + configurationDescriptor, + cacheCopyAttempt, + blacklistCache + ) - override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, - initializationData: Option[BackendInitializationData]): Option[Props] = { + override def workflowFinalizationActorProps(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, + initializationData: Option[BackendInitializationData] + ): Option[Props] = finalizationActorClassOption map { finalizationActorClass => - val params = workflowFinalizationActorParams(workflowDescriptor, ioActor, calls, jobExecutionMap, workflowOutputs, - initializationData) + val params = workflowFinalizationActorParams(workflowDescriptor, + ioActor, + calls, + jobExecutionMap, + workflowOutputs, + initializationData + ) Props(finalizationActorClass, params).withDispatcher(BackendDispatcher) } - } - def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, ioActor: ActorRef, calls: Set[CommandCallNode], - jobExecutionMap: JobExecutionMap, workflowOutputs: CallOutputs, - initializationDataOption: Option[BackendInitializationData]): - StandardFinalizationActorParams = { - DefaultStandardFinalizationActorParams(workflowDescriptor, calls, jobExecutionMap, workflowOutputs, - initializationDataOption, configurationDescriptor) - } + def workflowFinalizationActorParams(workflowDescriptor: BackendWorkflowDescriptor, + ioActor: ActorRef, + calls: Set[CommandCallNode], + jobExecutionMap: JobExecutionMap, + workflowOutputs: CallOutputs, + initializationDataOption: Option[BackendInitializationData] + ): StandardFinalizationActorParams = + DefaultStandardFinalizationActorParams(workflowDescriptor, + calls, + jobExecutionMap, + workflowOutputs, + initializationDataOption, + configurationDescriptor + ) override def expressionLanguageFunctions(workflowDescriptor: BackendWorkflowDescriptor, jobKey: BackendJobDescriptorKey, initializationDataOption: Option[BackendInitializationData], ioActorProxy: ActorRef, - ec: ExecutionContext): - IoFunctionSet = { + ec: ExecutionContext + ): IoFunctionSet = { val standardInitializationData = BackendInitializationData.as[StandardInitializationData](initializationDataOption) val jobPaths = standardInitializationData.workflowPaths.toJobPaths(jobKey, workflowDescriptor) standardInitializationData.expressionFunctions(jobPaths, ioActorProxy, ec) } - + override def pathBuilders(initializationDataOption: Option[BackendInitializationData]) = { val standardInitializationData = BackendInitializationData.as[StandardInitializationData](initializationDataOption) standardInitializationData.workflowPaths.pathBuilders - } + } - override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, - initializationData: Option[BackendInitializationData]): Path = { + override def getExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = initializationData match { case Some(data) => data.asInstanceOf[StandardInitializationData].workflowPaths.executionRoot case None => super.getExecutionRootPath(workflowDescriptor, backendConfig, initializationData) } - } - override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, backendConfig: Config, - initializationData: Option[BackendInitializationData]): Path = { + override def getWorkflowExecutionRootPath(workflowDescriptor: BackendWorkflowDescriptor, + backendConfig: Config, + initializationData: Option[BackendInitializationData] + ): Path = initializationData match { case Some(data) => data.asInstanceOf[StandardInitializationData].workflowPaths.workflowRoot case None => super.getWorkflowExecutionRootPath(workflowDescriptor, backendConfig, initializationData) } - } - override def runtimeAttributeDefinitions(initializationDataOption: Option[BackendInitializationData]): Set[RuntimeAttributeDefinition] = { - val initializationData = BackendInitializationData. - as[StandardInitializationData](initializationDataOption) + override def runtimeAttributeDefinitions( + initializationDataOption: Option[BackendInitializationData] + ): Set[RuntimeAttributeDefinition] = { + val initializationData = BackendInitializationData.as[StandardInitializationData](initializationDataOption) initializationData.runtimeAttributesBuilder.definitions.toSet } override def dockerHashCredentials(workflowDescriptor: BackendWorkflowDescriptor, - initializationDataOption: Option[BackendInitializationData], - ): List[Any] = { + initializationDataOption: Option[BackendInitializationData] + ): List[Any] = BackendDockerConfiguration.build(configurationDescriptor.backendConfig).dockerCredentials.toList - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala index 99e81fbf8b2..794602887e4 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardSyncExecutionActor.scala @@ -13,12 +13,12 @@ import scala.concurrent.{Future, Promise} import scala.util.control.NoStackTrace trait StandardSyncExecutionActorParams extends StandardJobExecutionActorParams { + /** The class for creating an async backend. */ def asyncJobExecutionActorClass: Class[_ <: StandardAsyncExecutionActor] } -case class DefaultStandardSyncExecutionActorParams -( +case class DefaultStandardSyncExecutionActorParams( override val jobIdKey: String, override val serviceRegistryActor: ActorRef, override val ioActor: ActorRef, @@ -52,7 +52,7 @@ case class DefaultStandardSyncExecutionActorParams * - Asynchronous actor completes the promise with a success or failure. */ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorParams) - extends BackendJobExecutionActor { + extends BackendJobExecutionActor { override val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor override val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor @@ -61,10 +61,9 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP context.become(startup orElse receive) - private def startup: Receive = { - case AbortJobCommand => - context.parent ! JobAbortedResponse(jobDescriptor.key) - context.stop(self) + private def startup: Receive = { case AbortJobCommand => + context.parent ! JobAbortedResponse(jobDescriptor.key) + context.stop(self) } private def running(executor: ActorRef): Receive = { @@ -78,7 +77,7 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP completionPromise.tryFailure(e) throw new RuntimeException(s"Failure attempting to look up job id for key ${jobDescriptor.key}", e) } - + private def recovering(executor: ActorRef): Receive = running(executor).orElse { case KvPair(key, jobId) if key.key == jobIdKey => // Successful operation ID lookup. @@ -131,20 +130,17 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP serviceRegistryActor ! kvGet completionPromise.future } - - override def recover: Future[BackendJobExecutionResponse] = { + + override def recover: Future[BackendJobExecutionResponse] = onRestart(recovering) - } - - override def reconnectToAborting: Future[BackendJobExecutionResponse] = { + + override def reconnectToAborting: Future[BackendJobExecutionResponse] = onRestart(reconnectingToAbort) - } - override def reconnect: Future[BackendJobExecutionResponse] = { + override def reconnect: Future[BackendJobExecutionResponse] = onRestart(reconnecting) - } - def createAsyncParams(): StandardAsyncExecutionActorParams = { + def createAsyncParams(): StandardAsyncExecutionActorParams = DefaultStandardAsyncExecutionActorParams( standardParams.jobIdKey, standardParams.serviceRegistryActor, @@ -156,16 +152,14 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP completionPromise, standardParams.minimumRuntimeSettings ) - } def createAsyncProps(): Props = { val asyncParams = createAsyncParams() Props(standardParams.asyncJobExecutionActorClass, asyncParams) } - def createAsyncRefName(): String = { + def createAsyncRefName(): String = standardParams.asyncJobExecutionActorClass.getSimpleName - } def createAsyncRef(): ActorRef = { val props = createAsyncProps().withDispatcher(Dispatcher.BackendDispatcher) @@ -173,16 +167,18 @@ class StandardSyncExecutionActor(val standardParams: StandardSyncExecutionActorP context.actorOf(props, name) } - override def abort(): Unit = { + override def abort(): Unit = throw new UnsupportedOperationException("Abort is implemented via a custom receive of the message AbortJobCommand.") - } // Supervision strategy: if the async actor throws an exception, stop the actor and fail the job. - def jobFailingDecider: Decider = { - case exception: Exception => - completionPromise.tryFailure( - new RuntimeException(s"${createAsyncRefName()} failed and didn't catch its exception. This condition has been handled and the job will be marked as failed.", exception) with NoStackTrace) - Stop + def jobFailingDecider: Decider = { case exception: Exception => + completionPromise.tryFailure( + new RuntimeException( + s"${createAsyncRefName()} failed and didn't catch its exception. This condition has been handled and the job will be marked as failed.", + exception + ) with NoStackTrace + ) + Stop } override val supervisorStrategy: OneForOneStrategy = OneForOneStrategy()(jobFailingDecider) diff --git a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala index 4160d9a8522..77f890b4a19 100644 --- a/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilder.scala @@ -17,8 +17,7 @@ import cromwell.backend.validation._ */ object StandardValidatedRuntimeAttributesBuilder { - private case class StandardValidatedRuntimeAttributesBuilderImpl - ( + private case class StandardValidatedRuntimeAttributesBuilderImpl( override val requiredValidations: Seq[RuntimeAttributesValidation[_]], override val customValidations: Seq[RuntimeAttributesValidation[_]] ) extends StandardValidatedRuntimeAttributesBuilder @@ -41,8 +40,8 @@ object StandardValidatedRuntimeAttributesBuilder { } private def withValidations(builder: StandardValidatedRuntimeAttributesBuilder, - customValidations: Seq[RuntimeAttributesValidation[_]]): - StandardValidatedRuntimeAttributesBuilder = { + customValidations: Seq[RuntimeAttributesValidation[_]] + ): StandardValidatedRuntimeAttributesBuilder = { val required = builder.requiredValidations val custom = builder.customValidations ++ customValidations StandardValidatedRuntimeAttributesBuilderImpl(custom, required) @@ -50,19 +49,18 @@ object StandardValidatedRuntimeAttributesBuilder { } sealed trait StandardValidatedRuntimeAttributesBuilder extends ValidatedRuntimeAttributesBuilder { + /** * Returns a new builder with the additional validation(s). * * @param validation Additional validation. * @return New builder with the validation. */ - final def withValidation(validation: RuntimeAttributesValidation[_]*): - StandardValidatedRuntimeAttributesBuilder = { + final def withValidation(validation: RuntimeAttributesValidation[_]*): StandardValidatedRuntimeAttributesBuilder = StandardValidatedRuntimeAttributesBuilder.withValidations(this, validation) - } /** Returns all the validations, those required for the standard backend, plus custom addons for the subclass. */ - override final lazy val validations: Seq[RuntimeAttributesValidation[_]] = requiredValidations ++ customValidations + final override lazy val validations: Seq[RuntimeAttributesValidation[_]] = requiredValidations ++ customValidations private[standard] def requiredValidations: Seq[RuntimeAttributesValidation[_]] diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala index ff3248123ff..fa5207c99c6 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/BlacklistCache.scala @@ -11,19 +11,20 @@ case object UntestedCacheResult extends BlacklistStatus sealed abstract class BlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig, - val name: Option[String]) { + val name: Option[String] +) { val bucketCache = { // Queries to the bucket blacklist cache return UntestedCacheResult by default. val unknownLoader = new CacheLoader[String, BlacklistStatus]() { override def load(key: String): BlacklistStatus = UntestedCacheResult } - CacheBuilder. - newBuilder(). - concurrencyLevel(bucketCacheConfig.concurrency). - maximumSize(bucketCacheConfig.size). - expireAfterWrite(bucketCacheConfig.ttl.length, bucketCacheConfig.ttl.unit). - build[String, BlacklistStatus](unknownLoader) + CacheBuilder + .newBuilder() + .concurrencyLevel(bucketCacheConfig.concurrency) + .maximumSize(bucketCacheConfig.size) + .expireAfterWrite(bucketCacheConfig.ttl.length, bucketCacheConfig.ttl.unit) + .build[String, BlacklistStatus](unknownLoader) } val hitCache = { @@ -32,12 +33,12 @@ sealed abstract class BlacklistCache(bucketCacheConfig: CacheConfig, override def load(key: CallCachingEntryId): BlacklistStatus = UntestedCacheResult } - CacheBuilder. - newBuilder(). - concurrencyLevel(hitCacheConfig.concurrency). - maximumSize(hitCacheConfig.size). - expireAfterWrite(hitCacheConfig.ttl.length, hitCacheConfig.ttl.unit). - build[CallCachingEntryId, BlacklistStatus](unknownLoader) + CacheBuilder + .newBuilder() + .concurrencyLevel(hitCacheConfig.concurrency) + .maximumSize(hitCacheConfig.size) + .expireAfterWrite(hitCacheConfig.ttl.length, hitCacheConfig.ttl.unit) + .build[CallCachingEntryId, BlacklistStatus](unknownLoader) } def getBlacklistStatus(hit: CallCachingEntryId): BlacklistStatus = hitCache.get(hit) @@ -53,8 +54,8 @@ sealed abstract class BlacklistCache(bucketCacheConfig: CacheConfig, def whitelist(bucket: String): Unit = bucketCache.put(bucket, GoodCacheResult) } -class RootWorkflowBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig) extends - BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = None) +class RootWorkflowBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig) + extends BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = None) -class GroupingBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig, val group: String) extends - BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = Option(group)) +class GroupingBlacklistCache(bucketCacheConfig: CacheConfig, hitCacheConfig: CacheConfig, val group: String) + extends BlacklistCache(bucketCacheConfig = bucketCacheConfig, hitCacheConfig = hitCacheConfig, name = Option(group)) diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala index a22631aeeb4..f1c6a988021 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManager.scala @@ -47,7 +47,7 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { import CallCachingBlacklistManager.Defaults.Groupings._ for { _ <- blacklistGroupingWorkflowOptionKey - groupingsOption = rootConfig.as[Option[Config]] ("call-caching.blacklist-cache.groupings") + groupingsOption = rootConfig.as[Option[Config]]("call-caching.blacklist-cache.groupings") conf = CacheConfig.config(groupingsOption, defaultConcurrency = Concurrency, defaultSize = Size, defaultTtl = Ttl) } yield conf } @@ -74,7 +74,10 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { // If configuration allows, build a cache of blacklist groupings to BlacklistCaches. private val blacklistGroupingsCache: Option[LoadingCache[String, BlacklistCache]] = { - def buildBlacklistGroupingsCache(groupingConfig: CacheConfig, bucketConfig: CacheConfig, hitConfig: CacheConfig): LoadingCache[String, BlacklistCache] = { + def buildBlacklistGroupingsCache(groupingConfig: CacheConfig, + bucketConfig: CacheConfig, + hitConfig: CacheConfig + ): LoadingCache[String, BlacklistCache] = { val emptyBlacklistCacheLoader = new CacheLoader[String, BlacklistCache]() { override def load(key: String): BlacklistCache = new GroupingBlacklistCache( bucketCacheConfig = bucketConfig, @@ -83,12 +86,12 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { ) } - CacheBuilder. - newBuilder(). - concurrencyLevel(groupingConfig.concurrency). - maximumSize(groupingConfig.size). - expireAfterWrite(groupingConfig.ttl.length, groupingConfig.ttl.unit). - build[String, BlacklistCache](emptyBlacklistCacheLoader) + CacheBuilder + .newBuilder() + .concurrencyLevel(groupingConfig.concurrency) + .maximumSize(groupingConfig.size) + .expireAfterWrite(groupingConfig.ttl.length, groupingConfig.ttl.unit) + .build[String, BlacklistCache](emptyBlacklistCacheLoader) } for { @@ -121,8 +124,13 @@ class CallCachingBlacklistManager(rootConfig: Config, logger: LoggingAdapter) { val maybeCache = groupBlacklistCache orElse rootWorkflowBlacklistCache maybeCache collect { case group: GroupingBlacklistCache => - logger.info("Workflow {} using group blacklist cache '{}' containing blacklist status for {} hits and {} buckets.", - workflow.id, group.group, group.hitCache.size(), group.bucketCache.size()) + logger.info( + "Workflow {} using group blacklist cache '{}' containing blacklist status for {} hits and {} buckets.", + workflow.id, + group.group, + group.hitCache.size(), + group.bucketCache.size() + ) case _: RootWorkflowBlacklistCache => logger.info("Workflow {} using root workflow blacklist cache.", workflow.id) } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala index 8ad88ae4f49..d90bba7ee90 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/CopyingActorBlacklistCacheSupport.scala @@ -4,7 +4,6 @@ import cromwell.backend.BackendCacheHitCopyingActor.CopyOutputsCommand import cromwell.core.io.{IoCommand, IoCopyCommand} import cromwell.services.CallCaching.CallCachingEntryId - object CopyingActorBlacklistCacheSupport { trait HasFormatting { def metricFormat: String = getClass.getName.toLowerCase.split('$').last @@ -52,13 +51,12 @@ trait CopyingActorBlacklistCacheSupport { } def publishBlacklistMetric(verb: Verb, entityType: EntityType, value: BlacklistStatus): Unit = { - val metricPath = NonEmptyList.of( - "job", - "callcaching", "blacklist", verb.metricFormat, entityType.metricFormat, value.toString) + val metricPath = + NonEmptyList.of("job", "callcaching", "blacklist", verb.metricFormat, entityType.metricFormat, value.toString) increment(metricPath) } - def blacklistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = { + def blacklistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = blacklistCache.getBlacklistStatus(hit) match { case UntestedCacheResult => blacklistCache.blacklist(hit) @@ -71,13 +69,13 @@ trait CopyingActorBlacklistCacheSupport { // mark the hit as BadCacheResult and log this strangeness. log.warning( "Cache hit {} found in GoodCacheResult blacklist state, but cache hit copying has failed for permissions reasons. Overwriting status to BadCacheResult state.", - hit.id) + hit.id + ) blacklistCache.blacklist(hit) publishBlacklistMetric(Write, Hit, value = BadCacheResult) } - } - def blacklistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = { + def blacklistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = blacklistCache.getBlacklistStatus(bucket) match { case UntestedCacheResult => blacklistCache.blacklist(bucket) @@ -90,13 +88,13 @@ trait CopyingActorBlacklistCacheSupport { // mark the bucket as BadCacheResult and log this strangeness. log.warning( "Bucket {} found in GoodCacheResult blacklist state, but cache hit copying has failed for permissions reasons. Overwriting status to BadCacheResult state.", - bucket) + bucket + ) blacklistCache.blacklist(bucket) publishBlacklistMetric(Write, Bucket, value = BadCacheResult) } - } - def whitelistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = { + def whitelistAndMetricHit(blacklistCache: BlacklistCache, hit: CallCachingEntryId): Unit = blacklistCache.getBlacklistStatus(hit) match { case UntestedCacheResult => blacklistCache.whitelist(hit) @@ -107,11 +105,11 @@ trait CopyingActorBlacklistCacheSupport { // Don't overwrite this to GoodCacheResult, hopefully there are less weird cache hits out there. log.warning( "Cache hit {} found in BadCacheResult blacklist state, not overwriting to GoodCacheResult despite successful copy.", - hit.id) + hit.id + ) } - } - def whitelistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = { + def whitelistAndMetricBucket(blacklistCache: BlacklistCache, bucket: String): Unit = blacklistCache.getBlacklistStatus(bucket) match { case UntestedCacheResult => blacklistCache.whitelist(bucket) @@ -122,11 +120,11 @@ trait CopyingActorBlacklistCacheSupport { // of a successful copy. Don't overwrite this to GoodCacheResult, hopefully there are less weird cache hits out there. log.warning( "Bucket {} found in BadCacheResult blacklist state, not overwriting to GoodCacheResult despite successful copy.", - bucket) + bucket + ) } - } - def publishBlacklistReadMetrics(command: CopyOutputsCommand, cacheHit: CallCachingEntryId, cacheReadType: Product) = { + def publishBlacklistReadMetrics(command: CopyOutputsCommand, cacheHit: CallCachingEntryId, cacheReadType: Product) = for { c <- standardParams.blacklistCache hitBlacklistStatus = c.getBlacklistStatus(cacheHit) @@ -139,7 +137,6 @@ trait CopyingActorBlacklistCacheSupport { bucketBlacklistStatus = c.getBlacklistStatus(prefix) _ = publishBlacklistMetric(Read, Bucket, bucketBlacklistStatus) } yield () - } def isSourceBlacklisted(command: CopyOutputsCommand): Boolean = { val path = sourcePathFromCopyOutputsCommand(command) @@ -150,10 +147,9 @@ trait CopyingActorBlacklistCacheSupport { } yield value == BadCacheResult).getOrElse(false) } - def isSourceBlacklisted(hit: CallCachingEntryId): Boolean = { + def isSourceBlacklisted(hit: CallCachingEntryId): Boolean = (for { cache <- standardParams.blacklistCache value = cache.getBlacklistStatus(hit) } yield value == BadCacheResult).getOrElse(false) - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala index e33625741b3..3ab77bdf778 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/RootWorkflowFileHashCacheActor.scala @@ -11,8 +11,10 @@ import cromwell.core.WorkflowId import cromwell.core.actor.RobustClientHelper.RequestTimeout import cromwell.core.io._ - -class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor: ActorRef, workflowId: WorkflowId) extends Actor with ActorLogging with IoClientHelper { +class RootWorkflowFileHashCacheActor private[callcaching] (override val ioActor: ActorRef, workflowId: WorkflowId) + extends Actor + with ActorLogging + with IoClientHelper { case class FileHashRequester(replyTo: ActorRef, fileHashContext: FileHashContext, ioCommand: IoCommand[_]) sealed trait FileHashValue @@ -25,8 +27,9 @@ class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor: // Hashing failed. case class FileHashFailure(error: String) extends FileHashValue - val cache: LoadingCache[String, FileHashValue] = CacheBuilder.newBuilder().build( - new CacheLoader[String, FileHashValue] { + val cache: LoadingCache[String, FileHashValue] = CacheBuilder + .newBuilder() + .build(new CacheLoader[String, FileHashValue] { override def load(key: String): FileHashValue = FileHashValueNotRequested }) @@ -48,7 +51,7 @@ class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor: // hash to become available. cache.put(key, FileHashValueRequested(requesters = requester :: requesters)) case FileHashSuccess(value) => - sender() ! Tuple2(hashCommand.fileHashContext, IoSuccess(requester.ioCommand, value)) + sender() ! Tuple2[Any, Any](hashCommand.fileHashContext, IoSuccess(requester.ioCommand, value)) case FileHashFailure(error) => sender() ! Tuple2(hashCommand.fileHashContext, IoFailure(requester.ioCommand, new IOException(error))) } @@ -66,44 +69,49 @@ class RootWorkflowFileHashCacheActor private[callcaching](override val ioActor: requesters foreach { case FileHashRequester(replyTo, fileHashContext, ioCommand) => replyTo ! Tuple2(fileHashContext, IoFailure(ioCommand, failure.failure)) } - cache.put(hashContext.file, FileHashFailure(s"Error hashing file '${hashContext.file}': ${failure.failure.getMessage}")) + cache.put(hashContext.file, + FileHashFailure(s"Error hashing file '${hashContext.file}': ${failure.failure.getMessage}") + ) } case other => log.warning(s"Root workflow file hash caching actor received unexpected message: $other") } // Invoke the supplied block on the happy path, handle unexpected states for IoSuccess and IoFailure with common code. - private def handleHashResult(ioAck: IoAck[_], fileHashContext: FileHashContext) - (notifyRequestersAndCacheValue: List[FileHashRequester] => Unit): Unit = { + private def handleHashResult(ioAck: IoAck[_], fileHashContext: FileHashContext)( + notifyRequestersAndCacheValue: List[FileHashRequester] => Unit + ): Unit = cache.get(fileHashContext.file) match { case FileHashValueRequested(requesters) => notifyRequestersAndCacheValue(requesters.toList) case FileHashValueNotRequested => log.info(msgIoAckWithNoRequesters.format(fileHashContext.file)) notifyRequestersAndCacheValue(List.empty[FileHashRequester]) case _ => - // IoAck arrived when hash result is already saved in cache. This is a result of benign race condition. - // No further action is required. + // IoAck arrived when hash result is already saved in cache. This is a result of benign race condition. + // No further action is required. } - } - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = message match { case (fileHashContext: FileHashContext, _) => // Send this message to all requestors. cache.get(fileHashContext.file) match { case FileHashValueRequested(requesters) => - requesters.toList foreach { case FileHashRequester(replyTo, requestContext, ioCommand) => replyTo ! RequestTimeout(Tuple2(requestContext, ioCommand), replyTo) } + requesters.toList foreach { case FileHashRequester(replyTo, requestContext, ioCommand) => + replyTo ! RequestTimeout(Tuple2(requestContext, ioCommand), replyTo) + } // Allow for the possibility of trying again on a timeout. cache.put(fileHashContext.file, FileHashValueNotRequested) case FileHashValueNotRequested => - // Due to race condition, timeout came after the actual response. This is fine and no further action required. + // Due to race condition, timeout came after the actual response. This is fine and no further action required. case v => log.info(msgTimeoutAfterIoAck.format(v, fileHashContext.file)) } case other => - log.error(s"Programmer error! Root workflow file hash caching actor received unexpected timeout message: $other") + log.error( + s"Programmer error! Root workflow file hash caching actor received unexpected timeout message: $other" + ) } - } override def preRestart(reason: Throwable, message: Option[Any]): Unit = { log.error(reason, s"RootWorkflowFileHashCacheActor for workflow '$workflowId' is unexpectedly being restarted") @@ -121,5 +129,7 @@ object RootWorkflowFileHashCacheActor { case class IoHashCommandWithContext(ioHashCommand: IoHashCommand, fileHashContext: FileHashContext) - def props(ioActor: ActorRef, workflowId: WorkflowId): Props = Props(new RootWorkflowFileHashCacheActor(ioActor, workflowId)) + def props(ioActor: ActorRef, workflowId: WorkflowId): Props = Props( + new RootWorkflowFileHashCacheActor(ioActor, workflowId) + ) } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala index f1662db858b..bded4b18f64 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardCacheHitCopyingActor.scala @@ -9,7 +9,12 @@ import cromwell.backend.io.JobPaths import cromwell.backend.standard.StandardCachingActorHelper import cromwell.backend.standard.callcaching.CopyingActorBlacklistCacheSupport._ import cromwell.backend.standard.callcaching.StandardCacheHitCopyingActor._ -import cromwell.backend.{BackendConfigurationDescriptor, BackendInitializationData, BackendJobDescriptor, MetricableCacheCopyErrorCategory} +import cromwell.backend.{ + BackendConfigurationDescriptor, + BackendInitializationData, + BackendJobDescriptor, + MetricableCacheCopyErrorCategory +} import cromwell.core.CallOutputs import cromwell.core.io._ import cromwell.core.logging.JobLogging @@ -46,8 +51,7 @@ trait StandardCacheHitCopyingActorParams { } /** A default implementation of the cache hit copying params. */ -case class DefaultStandardCacheHitCopyingActorParams -( +case class DefaultStandardCacheHitCopyingActorParams( override val jobDescriptor: BackendJobDescriptor, override val backendInitializationDataOption: Option[BackendInitializationData], override val serviceRegistryActor: ActorRef, @@ -81,32 +85,33 @@ object StandardCacheHitCopyingActor { newDetritus: DetritusMap, cacheHit: CallCachingEntryId, returnCode: Option[Int] - ) { + ) { /** * Removes the command from commandsToWaitFor * returns a pair of the new state data and CommandSetState giving information about what to do next */ - def commandComplete(command: IoCommand[_]): (StandardCacheHitCopyingActorData, CommandSetState) = commandsToWaitFor match { - // If everything was already done send back current data and AllCommandsDone - case Nil => (this, AllCommandsDone) - case lastSubset :: Nil => - val updatedSubset = lastSubset - command - // If the last subset is now empty, we're done - if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = List.empty), AllCommandsDone) - // otherwise update commandsToWaitFor and keep waiting - else (this.copy(commandsToWaitFor = List(updatedSubset)), StillWaiting) - case currentSubset :: otherSubsets => - val updatedSubset = currentSubset - command - // This subset is done but there are other ones, remove it from commandsToWaitFor and return the next round of commands - if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = otherSubsets), NextSubSet(otherSubsets.head)) - // otherwise update the head subset and keep waiting - else (this.copy(commandsToWaitFor = List(updatedSubset) ++ otherSubsets), StillWaiting) - } + def commandComplete(command: IoCommand[_]): (StandardCacheHitCopyingActorData, CommandSetState) = + commandsToWaitFor match { + // If everything was already done send back current data and AllCommandsDone + case Nil => (this, AllCommandsDone) + case lastSubset :: Nil => + val updatedSubset = lastSubset - command + // If the last subset is now empty, we're done + if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = List.empty), AllCommandsDone) + // otherwise update commandsToWaitFor and keep waiting + else (this.copy(commandsToWaitFor = List(updatedSubset)), StillWaiting) + case currentSubset :: otherSubsets => + val updatedSubset = currentSubset - command + // This subset is done but there are other ones, remove it from commandsToWaitFor and return the next round of commands + if (updatedSubset.isEmpty) (this.copy(commandsToWaitFor = otherSubsets), NextSubSet(otherSubsets.head)) + // otherwise update the head subset and keep waiting + else (this.copy(commandsToWaitFor = List(updatedSubset) ++ otherSubsets), StillWaiting) + } } // Internal ADT to keep track of command set states - private[callcaching] sealed trait CommandSetState + sealed private[callcaching] trait CommandSetState private[callcaching] case object StillWaiting extends CommandSetState private[callcaching] case object AllCommandsDone extends CommandSetState private[callcaching] case class NextSubSet(commands: Set[IoCommand[_]]) extends CommandSetState @@ -114,17 +119,23 @@ object StandardCacheHitCopyingActor { private val BucketRegex: Regex = "^gs://([^/]+).*".r } -class DefaultStandardCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams) extends StandardCacheHitCopyingActor(standardParams) +class DefaultStandardCacheHitCopyingActor(standardParams: StandardCacheHitCopyingActorParams) + extends StandardCacheHitCopyingActor(standardParams) /** * Standard implementation of a BackendCacheHitCopyingActor. */ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHitCopyingActorParams) - extends FSM[StandardCacheHitCopyingActorState, Option[StandardCacheHitCopyingActorData]] - with JobLogging with StandardCachingActorHelper with IoClientHelper with CromwellInstrumentationActor with CopyingActorBlacklistCacheSupport { + extends FSM[StandardCacheHitCopyingActorState, Option[StandardCacheHitCopyingActorData]] + with JobLogging + with StandardCachingActorHelper + with IoClientHelper + with CromwellInstrumentationActor + with CopyingActorBlacklistCacheSupport { override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor - override lazy val backendInitializationDataOption: Option[BackendInitializationData] = standardParams.backendInitializationDataOption + override lazy val backendInitializationDataOption: Option[BackendInitializationData] = + standardParams.backendInitializationDataOption override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor protected val commandBuilder: IoCommandBuilder = DefaultIoCommandBuilder @@ -142,78 +153,78 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** Override this method if you want to provide an alternative way to duplicate files than copying them. */ protected def duplicate(copyPairs: Set[PathPair]): Option[Try[Unit]] = None - when(Idle) { - case Event(command @ CopyOutputsCommand(simpletons, jobDetritus, cacheHit, returnCode), None) => - val (nextState, cacheReadType) = - if (isSourceBlacklisted(cacheHit)) { - // We don't want to log this because blacklisting is a common and expected occurrence. - (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.HitBlacklisted)), ReadHitOnly) - } else if (isSourceBlacklisted(command)) { - // We don't want to log this because blacklisting is a common and expected occurrence. - (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.BucketBlacklisted)), ReadHitAndBucket) - } else { - // Try to make a Path of the callRootPath from the detritus - val next = lookupSourceCallRootPath(jobDetritus) match { - case Success(sourceCallRootPath) => - - // process simpletons and detritus to get updated paths and corresponding IoCommands - val processed = for { - (destinationCallOutputs, simpletonIoCommands) <- processSimpletons(simpletons, sourceCallRootPath) - (destinationDetritus, detritusIoCommands) <- processDetritus(jobDetritus) - } yield (destinationCallOutputs, destinationDetritus, simpletonIoCommands ++ detritusIoCommands) - - processed match { - case Success((destinationCallOutputs, destinationDetritus, detritusAndOutputsIoCommands)) => - duplicate(ioCommandsToCopyPairs(detritusAndOutputsIoCommands)) match { - // Use the duplicate override if exists - case Some(Success(_)) => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) - case Some(Failure(failure)) => - // Something went wrong in the custom duplication code. We consider this loggable because it's most likely a user-permission error: - failAndStop(CopyAttemptError(failure)) - // Otherwise send the first round of IoCommands (file outputs and detritus) if any - case None if detritusAndOutputsIoCommands.nonEmpty => - detritusAndOutputsIoCommands foreach sendIoCommand - - // Add potential additional commands to the list - val additionalCommandsTry = - additionalIoCommands( - sourceCallRootPath = sourceCallRootPath, - originalSimpletons = simpletons, - newOutputs = destinationCallOutputs, - originalDetritus = jobDetritus, - newDetritus = destinationDetritus, - ) - additionalCommandsTry match { - case Success(additionalCommands) => - val allCommands = List(detritusAndOutputsIoCommands) ++ additionalCommands - goto(WaitingForIoResponses) using - Option(StandardCacheHitCopyingActorData( + when(Idle) { case Event(command @ CopyOutputsCommand(simpletons, jobDetritus, cacheHit, returnCode), None) => + val (nextState, cacheReadType) = + if (isSourceBlacklisted(cacheHit)) { + // We don't want to log this because blacklisting is a common and expected occurrence. + (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.HitBlacklisted)), ReadHitOnly) + } else if (isSourceBlacklisted(command)) { + // We don't want to log this because blacklisting is a common and expected occurrence. + (failAndStop(BlacklistSkip(MetricableCacheCopyErrorCategory.BucketBlacklisted)), ReadHitAndBucket) + } else { + // Try to make a Path of the callRootPath from the detritus + val next = lookupSourceCallRootPath(jobDetritus) match { + case Success(sourceCallRootPath) => + // process simpletons and detritus to get updated paths and corresponding IoCommands + val processed = for { + (destinationCallOutputs, simpletonIoCommands) <- processSimpletons(simpletons, sourceCallRootPath) + (destinationDetritus, detritusIoCommands) <- processDetritus(jobDetritus) + } yield (destinationCallOutputs, destinationDetritus, simpletonIoCommands ++ detritusIoCommands) + + processed match { + case Success((destinationCallOutputs, destinationDetritus, detritusAndOutputsIoCommands)) => + duplicate(ioCommandsToCopyPairs(detritusAndOutputsIoCommands)) match { + // Use the duplicate override if exists + case Some(Success(_)) => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) + case Some(Failure(failure)) => + // Something went wrong in the custom duplication code. We consider this loggable because it's most likely a user-permission error: + failAndStop(CopyAttemptError(failure)) + // Otherwise send the first round of IoCommands (file outputs and detritus) if any + case None if detritusAndOutputsIoCommands.nonEmpty => + detritusAndOutputsIoCommands foreach sendIoCommand + + // Add potential additional commands to the list + val additionalCommandsTry = + additionalIoCommands( + sourceCallRootPath = sourceCallRootPath, + originalSimpletons = simpletons, + newOutputs = destinationCallOutputs, + originalDetritus = jobDetritus, + newDetritus = destinationDetritus + ) + additionalCommandsTry match { + case Success(additionalCommands) => + val allCommands = List(detritusAndOutputsIoCommands) ++ additionalCommands + goto(WaitingForIoResponses) using + Option( + StandardCacheHitCopyingActorData( commandsToWaitFor = allCommands, newJobOutputs = destinationCallOutputs, newDetritus = destinationDetritus, cacheHit = cacheHit, - returnCode = returnCode, - )) - // Something went wrong in generating duplication commands. - // We consider this a loggable error because we don't expect this to happen: - case Failure(failure) => failAndStop(CopyAttemptError(failure)) - } - case _ => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) - } - - // Something went wrong in generating duplication commands. We consider this loggable error because we don't expect this to happen: - case Failure(failure) => failAndStop(CopyAttemptError(failure)) - } - - // Something went wrong in looking up the call root... loggable because we don't expect this to happen: - case Failure(failure) => failAndStop(CopyAttemptError(failure)) - } - (next, ReadHitAndBucket) + returnCode = returnCode + ) + ) + // Something went wrong in generating duplication commands. + // We consider this a loggable error because we don't expect this to happen: + case Failure(failure) => failAndStop(CopyAttemptError(failure)) + } + case _ => succeedAndStop(returnCode, destinationCallOutputs, destinationDetritus) + } + + // Something went wrong in generating duplication commands. We consider this loggable error because we don't expect this to happen: + case Failure(failure) => failAndStop(CopyAttemptError(failure)) + } + + // Something went wrong in looking up the call root... loggable because we don't expect this to happen: + case Failure(failure) => failAndStop(CopyAttemptError(failure)) } + (next, ReadHitAndBucket) + } - publishBlacklistReadMetrics(command, cacheHit, cacheReadType) + publishBlacklistReadMetrics(command, cacheHit, cacheReadType) - nextState + nextState } when(WaitingForIoResponses) { @@ -228,6 +239,10 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit data.commandsToWaitFor.flatten.headOption match { case Some(command: IoCopyCommand) => logCacheHitCopyCommand(command) + case Some(command: IoTouchCommand) => + logCacheHitTouchCommand(command) + case Some(command: IoWriteCommand) => + logCacheHitWriteCommand(command) case huh => log.warning(s"BT-322 {} unexpected commandsToWaitFor: {}", jobTag, huh) } @@ -293,7 +308,7 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit _ = blacklistAndMetricHit(cache, data.cacheHit) prefix <- extractBlacklistPrefix(path) _ = blacklistAndMetricBucket(cache, prefix) - } yield() + } yield () andThen } @@ -307,10 +322,26 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit case _ => } + private def logCacheHitTouchCommand(command: IoTouchCommand): Unit = + log.info(s"BT-322 {} cache touch hit for file : {}", jobTag, command.toString) + + private def logCacheHitWriteCommand(command: IoWriteCommand): Unit = + log.info(s"BT-322 {} cache write hit for file : {}", jobTag, command.toString) + def succeedAndStop(returnCode: Option[Int], copiedJobOutputs: CallOutputs, detritusMap: DetritusMap): State = { import cromwell.services.metadata.MetadataService.implicits.MetadataAutoPutter - serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, Option(jobDescriptor.key), startMetadataKeyValues) - context.parent ! JobSucceededResponse(jobDescriptor.key, returnCode, copiedJobOutputs, Option(detritusMap), Seq.empty, None, resultGenerationMode = CallCached) + serviceRegistryActor.putMetadata(jobDescriptor.workflowDescriptor.id, + Option(jobDescriptor.key), + startMetadataKeyValues + ) + context.parent ! JobSucceededResponse(jobDescriptor.key, + returnCode, + copiedJobOutputs, + Option(detritusMap), + Seq.empty, + None, + resultGenerationMode = CallCached + ) context stop self stay() } @@ -323,7 +354,10 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** If there are no responses pending this behaves like `failAndStop`, otherwise this goes to `FailedState` and waits * for all the pending responses to come back before stopping. */ - def failAndAwaitPendingResponses(failure: CacheCopyFailure, command: IoCommand[_], data: StandardCacheHitCopyingActorData): State = { + def failAndAwaitPendingResponses(failure: CacheCopyFailure, + command: IoCommand[_], + data: StandardCacheHitCopyingActorData + ): State = { context.parent ! CopyingOutputsFailedResponse(jobDescriptor.key, standardParams.cacheCopyAttempt, failure) val (newData, commandState) = data.commandComplete(command) @@ -344,12 +378,16 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit stay() } - protected def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Try[Path] = { + protected def lookupSourceCallRootPath(sourceJobDetritusFiles: Map[String, String]): Try[Path] = sourceJobDetritusFiles.get(JobPaths.CallRootPathKey) match { case Some(source) => getPath(source) - case None => Failure(new RuntimeException(s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.taskCall.fullyQualifiedName}")) + case None => + Failure( + new RuntimeException( + s"${JobPaths.CallRootPathKey} wasn't found for call ${jobDescriptor.taskCall.fullyQualifiedName}" + ) + ) } - } private def ioCommandsToCopyPairs(commands: Set[IoCommand[_]]): Set[PathPair] = commands collect { case copyCommand: IoCopyCommand => copyCommand.source -> copyCommand.destination @@ -358,18 +396,22 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** * Returns a pair of the list of simpletons with copied paths, and copy commands necessary to perform those copies. */ - protected def processSimpletons(womValueSimpletons: Seq[WomValueSimpleton], sourceCallRootPath: Path): Try[(CallOutputs, Set[IoCommand[_]])] = Try { - val (destinationSimpletons, ioCommands): (List[WomValueSimpleton], Set[IoCommand[_]]) = womValueSimpletons.toList.foldMap({ - case WomValueSimpleton(key, wdlFile: WomSingleFile) => - val sourcePath = getPath(wdlFile.value).get - val destinationPath = PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath) - - val destinationSimpleton = WomValueSimpleton(key, WomSingleFile(destinationPath.pathAsString)) - - // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap - List(destinationSimpleton) -> Set(commandBuilder.copyCommand(sourcePath, destinationPath).get) - case nonFileSimpleton => (List(nonFileSimpleton), Set.empty[IoCommand[_]]) - }) + protected def processSimpletons(womValueSimpletons: Seq[WomValueSimpleton], + sourceCallRootPath: Path + ): Try[(CallOutputs, Set[IoCommand[_]])] = Try { + val (destinationSimpletons, ioCommands): (List[WomValueSimpleton], Set[IoCommand[_]]) = + womValueSimpletons.toList.foldMap { + case WomValueSimpleton(key, wdlFile: WomSingleFile) => + val sourcePath = getPath(wdlFile.value).get + val destinationPath = + PathCopier.getDestinationFilePath(sourceCallRootPath, sourcePath, destinationCallRootPath) + + val destinationSimpleton = WomValueSimpleton(key, WomSingleFile(destinationPath.pathAsString)) + + // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap + List(destinationSimpleton) -> Set(commandBuilder.copyCommand(sourcePath, destinationPath).get) + case nonFileSimpleton => (List(nonFileSimpleton), Set.empty[IoCommand[_]]) + } (WomValueBuilder.toJobOutputs(jobDescriptor.taskCall.outputPorts, destinationSimpletons), ioCommands) } @@ -377,7 +419,7 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** * Returns the file (and ONLY the file detritus) intersection between the cache hit and this call. */ - protected final def detritusFileKeys(sourceJobDetritusFiles: Map[String, String]): Set[String] = { + final protected def detritusFileKeys(sourceJobDetritusFiles: Map[String, String]): Set[String] = { val sourceKeys = sourceJobDetritusFiles.keySet val destinationKeys = destinationJobDetritusPaths.keySet sourceKeys.intersect(destinationKeys).filterNot(_ == JobPaths.CallRootPathKey) @@ -386,21 +428,22 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit /** * Returns a pair of the detritus with copied paths, and copy commands necessary to perform those copies. */ - protected def processDetritus(sourceJobDetritusFiles: Map[String, String]): Try[(Map[String, Path], Set[IoCommand[_]])] = Try { + protected def processDetritus( + sourceJobDetritusFiles: Map[String, String] + ): Try[(Map[String, Path], Set[IoCommand[_]])] = Try { val fileKeys = detritusFileKeys(sourceJobDetritusFiles) val zero = (Map.empty[String, Path], Set.empty[IoCommand[_]]) - val (destinationDetritus, ioCommands) = fileKeys.foldLeft(zero)({ - case ((detrituses, commands), detritus) => - val sourcePath = getPath(sourceJobDetritusFiles(detritus)).get - val destinationPath = destinationJobDetritusPaths(detritus) + val (destinationDetritus, ioCommands) = fileKeys.foldLeft(zero) { case ((detrituses, commands), detritus) => + val sourcePath = getPath(sourceJobDetritusFiles(detritus)).get + val destinationPath = destinationJobDetritusPaths(detritus) - val newDetrituses = detrituses + (detritus -> destinationPath) + val newDetrituses = detrituses + (detritus -> destinationPath) // PROD-444: Keep It Short and Simple: Throw on the first error and let the outer Try catch-and-re-wrap (newDetrituses, commands + commandBuilder.copyCommand(sourcePath, destinationPath).get) - }) + } (destinationDetritus + (JobPaths.CallRootPathKey -> destinationCallRootPath), ioCommands) } @@ -412,13 +455,16 @@ abstract class StandardCacheHitCopyingActor(val standardParams: StandardCacheHit protected def additionalIoCommands(sourceCallRootPath: Path, originalSimpletons: Seq[WomValueSimpleton], newOutputs: CallOutputs, - originalDetritus: Map[String, String], - newDetritus: Map[String, Path]): Try[List[Set[IoCommand[_]]]] = Success(Nil) + originalDetritus: Map[String, String], + newDetritus: Map[String, Path] + ): Try[List[Set[IoCommand[_]]]] = Success(Nil) override protected def onTimeout(message: Any, to: ActorRef): Unit = { val exceptionMessage = message match { - case copyCommand: IoCopyCommand => s"The Cache hit copying actor timed out waiting for a response to copy ${copyCommand.source.pathAsString} to ${copyCommand.destination.pathAsString}" - case touchCommand: IoTouchCommand => s"The Cache hit copying actor timed out waiting for a response to touch ${touchCommand.file.pathAsString}" + case copyCommand: IoCopyCommand => + s"The Cache hit copying actor timed out waiting for a response to copy ${copyCommand.source.pathAsString} to ${copyCommand.destination.pathAsString}" + case touchCommand: IoTouchCommand => + s"The Cache hit copying actor timed out waiting for a response to touch ${touchCommand.file.pathAsString}" case other => s"The Cache hit copying actor timed out waiting for an unknown I/O operation: $other" } diff --git a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala index 78fdc078cb9..67199a7a062 100644 --- a/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala +++ b/backend/src/main/scala/cromwell/backend/standard/callcaching/StandardFileHashingActor.scala @@ -34,8 +34,7 @@ trait StandardFileHashingActorParams { } /** A default implementation of the cache hit copying params. */ -case class DefaultStandardFileHashingActorParams -( +case class DefaultStandardFileHashingActorParams( override val jobDescriptor: BackendJobDescriptor, override val backendInitializationDataOption: Option[BackendInitializationData], override val serviceRegistryActor: ActorRef, @@ -46,7 +45,8 @@ case class DefaultStandardFileHashingActorParams case class FileHashContext(hashKey: HashKey, file: String) -class DefaultStandardFileHashingActor(standardParams: StandardFileHashingActorParams) extends StandardFileHashingActor(standardParams) { +class DefaultStandardFileHashingActor(standardParams: StandardFileHashingActorParams) + extends StandardFileHashingActor(standardParams) { override val ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder } @@ -54,14 +54,20 @@ object StandardFileHashingActor { case class FileHashingFunction(work: (SingleFileHashRequest, LoggingAdapter) => Try[String]) sealed trait BackendSpecificHasherCommand { def jobKey: JobKey } - final case class SingleFileHashRequest(jobKey: JobKey, hashKey: HashKey, file: WomFile, initializationData: Option[BackendInitializationData]) extends BackendSpecificHasherCommand + final case class SingleFileHashRequest(jobKey: JobKey, + hashKey: HashKey, + file: WomFile, + initializationData: Option[BackendInitializationData] + ) extends BackendSpecificHasherCommand sealed trait BackendSpecificHasherResponse extends SuccessfulHashResultMessage - case class FileHashResponse(hashResult: HashResult) extends BackendSpecificHasherResponse { override def hashes = Set(hashResult) } + case class FileHashResponse(hashResult: HashResult) extends BackendSpecificHasherResponse { + override def hashes = Set(hashResult) + } } abstract class StandardFileHashingActor(standardParams: StandardFileHashingActorParams) - extends Actor + extends Actor with ActorLogging with JobLogging with IoClientHelper @@ -69,19 +75,21 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor with Timers { override lazy val ioActor: ActorRef = standardParams.ioActor override lazy val jobDescriptor: BackendJobDescriptor = standardParams.jobDescriptor - override lazy val backendInitializationDataOption: Option[BackendInitializationData] = standardParams.backendInitializationDataOption + override lazy val backendInitializationDataOption: Option[BackendInitializationData] = + standardParams.backendInitializationDataOption override lazy val serviceRegistryActor: ActorRef = standardParams.serviceRegistryActor override lazy val configurationDescriptor: BackendConfigurationDescriptor = standardParams.configurationDescriptor protected def ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder def customHashStrategy(fileRequest: SingleFileHashRequest): Option[Try[String]] = None - + def fileHashingReceive: Receive = { // Hash Request case fileRequest: SingleFileHashRequest => customHashStrategy(fileRequest) match { - case Some(Success(result)) => context.parent ! FileHashResponse(HashResult(fileRequest.hashKey, HashValue(result))) + case Some(Success(result)) => + context.parent ! FileHashResponse(HashResult(fileRequest.hashKey, HashValue(result))) case Some(Failure(failure)) => context.parent ! HashingFailedMessage(fileRequest.file.value, failure) case None => asyncHashing(fileRequest, context.parent) } @@ -93,7 +101,7 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor case (fileHashRequest: FileHashContext, IoSuccess(_, other)) => context.parent ! HashingFailedMessage( fileHashRequest.file, - new Exception(s"Hash function supposedly succeeded but responded with '$other' instead of a string hash"), + new Exception(s"Hash function supposedly succeeded but responded with '$other' instead of a string hash") ) // Hash Failure @@ -124,9 +132,9 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor } } - override def receive: Receive = ioReceive orElse fileHashingReceive + override def receive: Receive = ioReceive orElse fileHashingReceive - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = message match { case (_, ioHashCommand: IoHashCommand) => val fileAsString = ioHashCommand.file.pathAsString @@ -137,5 +145,4 @@ abstract class StandardFileHashingActor(standardParams: StandardFileHashingActor log.warning(s"Async File hashing actor received unexpected timeout message: $other") context.parent ! HashingServiceUnvailable } - } } diff --git a/backend/src/main/scala/cromwell/backend/standard/package.scala b/backend/src/main/scala/cromwell/backend/standard/package.scala index 089efd4f614..b25d74ec8e7 100644 --- a/backend/src/main/scala/cromwell/backend/standard/package.scala +++ b/backend/src/main/scala/cromwell/backend/standard/package.scala @@ -12,7 +12,7 @@ package object standard { def unapply(arg: StandardAdHocValue): Option[LocalizedAdHocValue] = arg.select[LocalizedAdHocValue] } } - + // This is used to represent an AdHocValue that might have been localized type StandardAdHocValue = AdHocValue :+: LocalizedAdHocValue :+: CNil } diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala index 06bc3d56c0a..c314a660122 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCode.scala @@ -10,18 +10,18 @@ object ContinueOnReturnCode { * Decides if a call/job continues upon a specific return code. */ sealed trait ContinueOnReturnCode { + /** * Returns true if the call is a success based on the return code. * * @param returnCode Return code from the process / script. * @return True if the call is a success. */ - final def continueFor(returnCode: Int): Boolean = { + final def continueFor(returnCode: Int): Boolean = this match { case ContinueOnReturnCodeFlag(continue) => continue || returnCode == 0 case ContinueOnReturnCodeSet(returnCodes) => returnCodes.contains(returnCode) } - } } /** diff --git a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala index aaa2754f3e1..7573909eb69 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ContinueOnReturnCodeValidation.scala @@ -3,8 +3,8 @@ package cromwell.backend.validation import cats.data.Validated.{Invalid, Valid} import cats.implicits._ import com.typesafe.config.Config -import cromwell.backend.validation.RuntimeAttributesValidation._ import common.validation.ErrorOr._ +import cromwell.backend.validation.RuntimeAttributesValidation.validateInt import wom.RuntimeAttributesKeys import wom.types._ import wom.values._ @@ -23,10 +23,14 @@ import scala.util.Try * `default` a validation with the default value specified by the reference.conf file. */ object ContinueOnReturnCodeValidation { - lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = new ContinueOnReturnCodeValidation - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[ContinueOnReturnCode] = instance.withDefault( - configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) - def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(runtimeConfig) + lazy val instance: RuntimeAttributesValidation[ContinueOnReturnCode] = + new ContinueOnReturnCodeValidation + def default( + runtimeConfig: Option[Config] + ): RuntimeAttributesValidation[ContinueOnReturnCode] = + instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomInteger(0)) + def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = + instance.configDefaultWomValue(runtimeConfig) } class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[ContinueOnReturnCode] { @@ -38,30 +42,34 @@ class ContinueOnReturnCodeValidation extends RuntimeAttributesValidation[Continu override def validateValue: PartialFunction[WomValue, ErrorOr[ContinueOnReturnCode]] = { case WomBoolean(value) => ContinueOnReturnCodeFlag(value).validNel case WomString(value) if Try(value.toBoolean).isSuccess => ContinueOnReturnCodeFlag(value.toBoolean).validNel + case WomString(value) if value.equals("*") => ContinueOnReturnCodeFlag(true).validNel case WomString(value) if Try(value.toInt).isSuccess => ContinueOnReturnCodeSet(Set(value.toInt)).validNel case WomInteger(value) => ContinueOnReturnCodeSet(Set(value)).validNel - case value@WomArray(_, seq) => + case value @ WomArray(_, seq) => val errorOrInts: ErrorOr[List[Int]] = (seq.toList map validateInt).sequence[ErrorOr, Int] errorOrInts match { case Valid(ints) => ContinueOnReturnCodeSet(ints.toSet).validNel case Invalid(_) => invalidValueFailure(value) } + case value => invalidValueFailure(value) } override def validateExpression: PartialFunction[WomValue, Boolean] = { case WomBoolean(_) => true case WomString(value) if Try(value.toInt).isSuccess => true case WomString(value) if Try(value.toBoolean).isSuccess => true + case WomString(value) if value.equals("*") => true case WomInteger(_) => true case WomArray(WomArrayType(WomStringType), elements) => - elements forall { - value => Try(value.valueString.toInt).isSuccess + elements forall { value => + Try(value.valueString.toInt).isSuccess } case WomArray(WomArrayType(WomIntegerType), _) => true + case _ => false } - override protected def missingValueMessage: String = s"Expecting $key" + - " runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + override protected def missingValueMessage: String = "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." override def usedInCallCaching: Boolean = true } diff --git a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala index 0cc06a7dfa4..8377213b42f 100644 --- a/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/CpuValidation.scala @@ -28,7 +28,7 @@ object CpuValidation { lazy val optionalMin: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMin.optional lazy val instanceMax: RuntimeAttributesValidation[Int Refined Positive] = new CpuValidation(CpuMaxKey) lazy val optionalMax: OptionalRuntimeAttributesValidation[Int Refined Positive] = instanceMax.optional - + lazy val defaultMin: WomValue = WomInteger(1) def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) } diff --git a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala index b6c3fe0112b..eb8afc17c12 100644 --- a/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/DockerValidation.scala @@ -26,7 +26,7 @@ class DockerValidation extends StringRuntimeAttributesValidation(RuntimeAttribut override protected def invalidValueMessage(value: WomValue): String = super.missingValueMessage // NOTE: Docker's current test specs don't like WdlInteger, etc. auto converted to WdlString. - override protected def validateValue: PartialFunction[WomValue, ErrorOr[String]] = { - case WomString(value) => value.validNel + override protected def validateValue: PartialFunction[WomValue, ErrorOr[String]] = { case WomString(value) => + value.validNel } } diff --git a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala index 8ac39e50d16..7b55f9657fa 100644 --- a/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/FailOnStderrValidation.scala @@ -18,9 +18,10 @@ import wom.values._ object FailOnStderrValidation { lazy val instance: RuntimeAttributesValidation[Boolean] = new FailOnStderrValidation - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = instance.withDefault( - configDefaultWdlValue(runtimeConfig) getOrElse WomBoolean(false)) - def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(runtimeConfig) + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Boolean] = + instance.withDefault(configDefaultWdlValue(runtimeConfig) getOrElse WomBoolean(false)) + def configDefaultWdlValue(runtimeConfig: Option[Config]): Option[WomValue] = + instance.configDefaultWomValue(runtimeConfig) } class FailOnStderrValidation extends BooleanRuntimeAttributesValidation(RuntimeAttributesKeys.FailOnStderrKey) { diff --git a/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala b/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala index 39b9f0b3031..e8c9dd1d7b3 100644 --- a/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/InformationValidation.scala @@ -25,18 +25,32 @@ import scala.util.{Failure, Success} * `withDefault` can be used to create a validation that defaults to a particular size. */ object InformationValidation { - def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey, defaultUnit: MemoryUnit, allowZero: Boolean = false): RuntimeAttributesValidation[MemorySize] = + def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey, + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): RuntimeAttributesValidation[MemorySize] = new InformationValidation(attributeName, defaultUnit, allowZero) - def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey, defaultUnit: MemoryUnit, allowZero: Boolean = false): OptionalRuntimeAttributesValidation[MemorySize] = + def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey, + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): OptionalRuntimeAttributesValidation[MemorySize] = instance(attributeName, defaultUnit, allowZero).optional - def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, config: Option[Config], defaultUnit: MemoryUnit, allowZero: Boolean = false): Option[String] = + def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, + config: Option[Config], + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): Option[String] = instance(attributeName, defaultUnit, allowZero).configDefaultValue(config) - def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, memorySize: String, defaultUnit: MemoryUnit, allowZero: Boolean = false): RuntimeAttributesValidation[MemorySize] = { + def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, + memorySize: String, + defaultUnit: MemoryUnit, + allowZero: Boolean = false + ): RuntimeAttributesValidation[MemorySize] = MemorySize.parse(memorySize) match { case Success(memory) => instance(attributeName, defaultUnit, allowZero).withDefault(WomLong(memory.bytes.toLong)) - case Failure(_) => instance(attributeName, defaultUnit, allowZero).withDefault(BadDefaultAttribute(WomString(memorySize.toString))) + case Failure(_) => + instance(attributeName, defaultUnit, allowZero).withDefault(BadDefaultAttribute(WomString(memorySize.toString))) } - } private[validation] val wrongAmountFormat = "Expecting %s runtime attribute value greater than 0 but got %s" @@ -44,39 +58,56 @@ object InformationValidation { "Expecting %s runtime attribute to be an Integer or String with format '8 GB'." + " Exception: %s" - private[validation] def validateString(attributeName: String, wdlString: WomString, allowZero: Boolean): ErrorOr[MemorySize] = + private[validation] def validateString(attributeName: String, + wdlString: WomString, + allowZero: Boolean + ): ErrorOr[MemorySize] = validateString(attributeName, wdlString.value, allowZero) - private[validation] def validateString(attributeName: String, value: String, allowZero: Boolean): ErrorOr[MemorySize] = { + private[validation] def validateString(attributeName: String, + value: String, + allowZero: Boolean + ): ErrorOr[MemorySize] = MemorySize.parse(value) match { - case scala.util.Success(memorySize: MemorySize) if memorySize.amount > 0 || (memorySize.amount == 0 && allowZero) => + case scala.util.Success(memorySize: MemorySize) + if memorySize.amount > 0 || (memorySize.amount == 0 && allowZero) => memorySize.to(MemoryUnit.GB).validNel case scala.util.Success(memorySize: MemorySize) => wrongAmountFormat.format(attributeName, memorySize.amount).invalidNel case scala.util.Failure(throwable) => wrongTypeFormat.format(attributeName, throwable.getMessage).invalidNel } - } - private[validation] def validateInteger(attributeName: String, wdlInteger: WomInteger, defaultUnit: MemoryUnit, allowZero: Boolean): ErrorOr[MemorySize] = + private[validation] def validateInteger(attributeName: String, + wdlInteger: WomInteger, + defaultUnit: MemoryUnit, + allowZero: Boolean + ): ErrorOr[MemorySize] = validateInteger(attributeName, wdlInteger.value, defaultUnit, allowZero) - private[validation] def validateInteger(attributeName: String, value: Int, defaultUnit: MemoryUnit, allowZero: Boolean): ErrorOr[MemorySize] = { + private[validation] def validateInteger(attributeName: String, + value: Int, + defaultUnit: MemoryUnit, + allowZero: Boolean + ): ErrorOr[MemorySize] = if (value < 0 || (value == 0 && !allowZero)) wrongAmountFormat.format(attributeName, value).invalidNel else MemorySize(value.toDouble, defaultUnit).to(MemoryUnit.GB).validNel - } - def validateLong(attributeName: String, value: Long, defaultUnit: MemoryUnit, allowZero: Boolean): ErrorOr[MemorySize] = { + def validateLong(attributeName: String, + value: Long, + defaultUnit: MemoryUnit, + allowZero: Boolean + ): ErrorOr[MemorySize] = if (value < 0 || (value == 0 && !allowZero)) wrongAmountFormat.format(attributeName, value).invalidNel else MemorySize(value.toDouble, defaultUnit).to(MemoryUnit.GB).validNel - } } -class InformationValidation(attributeName: String, defaultUnit: MemoryUnit, allowZero: Boolean = false) extends RuntimeAttributesValidation[MemorySize] { +class InformationValidation(attributeName: String, defaultUnit: MemoryUnit, allowZero: Boolean = false) + extends RuntimeAttributesValidation[MemorySize] { import InformationValidation._ diff --git a/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala index fdfd6467a85..f2c9ba9522e 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MaxRetriesValidation.scala @@ -21,8 +21,8 @@ object MaxRetriesValidation { lazy val instance: RuntimeAttributesValidation[Int] = new MaxRetriesValidation(MaxRetriesKey) lazy val optional: OptionalRuntimeAttributesValidation[Int] = instance.optional - def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = instance.withDefault( - configDefaultWomValue(runtimeConfig) getOrElse WomInteger(0)) + def default(runtimeConfig: Option[Config]): RuntimeAttributesValidation[Int] = + instance.withDefault(configDefaultWomValue(runtimeConfig) getOrElse WomInteger(0)) def configDefaultWomValue(config: Option[Config]): Option[WomValue] = instance.configDefaultWomValue(config) } diff --git a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala index 32be28026f0..299e043ec8f 100644 --- a/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/MemoryValidation.scala @@ -25,16 +25,22 @@ import scala.util.{Failure, Success} object MemoryValidation { def instance(attributeName: String = RuntimeAttributesKeys.MemoryKey): RuntimeAttributesValidation[MemorySize] = new MemoryValidation(attributeName) - def optional(attributeName: String = RuntimeAttributesKeys.MemoryKey): OptionalRuntimeAttributesValidation[MemorySize] = + def optional( + attributeName: String = RuntimeAttributesKeys.MemoryKey + ): OptionalRuntimeAttributesValidation[MemorySize] = instance(attributeName).optional - def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, config: Option[Config]): Option[String] = + def configDefaultString(attributeName: String = RuntimeAttributesKeys.MemoryKey, + config: Option[Config] + ): Option[String] = instance(attributeName).configDefaultValue(config) - def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, memorySize: String): RuntimeAttributesValidation[MemorySize] = { + def withDefaultMemory(attributeName: String = RuntimeAttributesKeys.MemoryKey, + memorySize: String + ): RuntimeAttributesValidation[MemorySize] = MemorySize.parse(memorySize) match { case Success(memory) => instance(attributeName).withDefault(WomLong(memory.bytes.toLong)) case Failure(_) => instance(attributeName).withDefault(BadDefaultAttribute(WomString(memorySize.toString))) } - } } -class MemoryValidation(attributeName: String = RuntimeAttributesKeys.MemoryKey) extends InformationValidation(attributeName, MemoryUnit.Bytes) +class MemoryValidation(attributeName: String = RuntimeAttributesKeys.MemoryKey) + extends InformationValidation(attributeName, MemoryUnit.Bytes) diff --git a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala index d4dc78f83b2..af41bf8ad52 100644 --- a/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/PrimitiveRuntimeAttributesValidation.scala @@ -44,24 +44,24 @@ sealed trait PrimitiveRuntimeAttributesValidation[A, B <: WomPrimitive] extends protected def validateCoercedValue(womValue: B): ErrorOr[A] } -class BooleanRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Boolean, WomBoolean] { +class BooleanRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Boolean, WomBoolean] { override val womType = WomBooleanType override protected def validateCoercedValue(womValue: WomBoolean): ErrorOr[Boolean] = womValue.value.validNel } -class FloatRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Double, WomFloat] { +class FloatRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Double, WomFloat] { override val womType = WomFloatType override protected def validateCoercedValue(womValue: WomFloat): ErrorOr[Double] = womValue.value.validNel } -class IntRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Int, WomInteger] { +class IntRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Int, WomInteger] { override val womType = WomIntegerType @@ -70,18 +70,19 @@ class IntRuntimeAttributesValidation(override val key: String) extends override protected def typeString: String = "an Integer" } -class PositiveIntRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[Int Refined Positive, WomInteger] { +class PositiveIntRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[Int Refined Positive, WomInteger] { override val womType = WomIntegerType - override protected def validateCoercedValue(womValue: WomInteger): ErrorOr[Int Refined Positive] = refineV[Positive](womValue.value).leftMap(NonEmptyList.one).toValidated + override protected def validateCoercedValue(womValue: WomInteger): ErrorOr[Int Refined Positive] = + refineV[Positive](womValue.value).leftMap(NonEmptyList.one).toValidated override protected def typeString: String = "an Integer" } -class StringRuntimeAttributesValidation(override val key: String) extends - PrimitiveRuntimeAttributesValidation[String, WomString] { +class StringRuntimeAttributesValidation(override val key: String) + extends PrimitiveRuntimeAttributesValidation[String, WomString] { override val womType = WomStringType diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala index d46bc7a66d5..706cc1336e8 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesDefault.scala @@ -12,29 +12,32 @@ import scala.util.{Failure, Try} object RuntimeAttributesDefault { - def workflowOptionsDefault(options: WorkflowOptions, mapping: Map[String, Iterable[WomType]]): - Try[Map[String, WomValue]] = { + def workflowOptionsDefault(options: WorkflowOptions, + mapping: Map[String, Iterable[WomType]] + ): Try[Map[String, WomValue]] = options.defaultRuntimeOptions flatMap { attrs => - TryUtil.sequenceMap(attrs collect { - case (k, v) if mapping.contains(k) => - val maybeTriedValue = mapping(k) map { _.coerceRawValue(v) } find { _.isSuccess } getOrElse { - Failure(new RuntimeException(s"Could not parse JsonValue $v to valid WomValue for runtime attribute $k")) - } - k -> maybeTriedValue - }, "Failed to coerce default runtime options") - } recover { - case _: OptionNotFoundException => Map.empty[String, WomValue] + TryUtil.sequenceMap( + attrs collect { + case (k, v) if mapping.contains(k) => + val maybeTriedValue = mapping(k) map { _.coerceRawValue(v) } find { _.isSuccess } getOrElse { + Failure(new RuntimeException(s"Could not parse JsonValue $v to valid WomValue for runtime attribute $k")) + } + k -> maybeTriedValue + }, + "Failed to coerce default runtime options" + ) + } recover { case _: OptionNotFoundException => + Map.empty[String, WomValue] } - } /** * Traverse defaultsList in order, and for each of them add the missing (and only missing) runtime attributes. */ - def withDefaults(attrs: EvaluatedRuntimeAttributes, defaultsList: List[EvaluatedRuntimeAttributes]): EvaluatedRuntimeAttributes = { - defaultsList.foldLeft(attrs)((acc, default) => { - acc ++ default.view.filterKeys(!acc.keySet.contains(_)) - }) - } + def withDefaults(attrs: EvaluatedRuntimeAttributes, + defaultsList: List[EvaluatedRuntimeAttributes] + ): EvaluatedRuntimeAttributes = + defaultsList.foldLeft(attrs)((acc, default) => acc ++ default.view.filterKeys(!acc.keySet.contains(_))) - def noValueFoundFor[A](attribute: String): ValidatedNel[String, A] = s"Can't find an attribute value for key $attribute".invalidNel + def noValueFoundFor[A](attribute: String): ValidatedNel[String, A] = + s"Can't find an attribute value for key $attribute".invalidNel } diff --git a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala index 65f2119a64b..5fa52dac53a 100644 --- a/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala +++ b/backend/src/main/scala/cromwell/backend/validation/RuntimeAttributesValidation.scala @@ -26,61 +26,56 @@ object RuntimeAttributesValidation { if (unrecognized.nonEmpty) logger.warn(s"Unrecognized runtime attribute keys: $unrecognized") } - def validateDocker(docker: Option[WomValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = { + def validateDocker(docker: Option[WomValue], onMissingKey: => ErrorOr[Option[String]]): ErrorOr[Option[String]] = validateWithValidation(docker, DockerValidation.instance.optional, onMissingKey) - } - def validateFailOnStderr(value: Option[WomValue], onMissingKey: => ErrorOr[Boolean]): ErrorOr[Boolean] = { + def validateFailOnStderr(value: Option[WomValue], onMissingKey: => ErrorOr[Boolean]): ErrorOr[Boolean] = validateWithValidation(value, FailOnStderrValidation.instance, onMissingKey) - } def validateContinueOnReturnCode(value: Option[WomValue], - onMissingKey: => ErrorOr[ContinueOnReturnCode]): ErrorOr[ContinueOnReturnCode] = { + onMissingKey: => ErrorOr[ContinueOnReturnCode] + ): ErrorOr[ContinueOnReturnCode] = validateWithValidation(value, ContinueOnReturnCodeValidation.instance, onMissingKey) - } - def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = { + def validateMemory(value: Option[WomValue], onMissingKey: => ErrorOr[MemorySize]): ErrorOr[MemorySize] = validateWithValidation(value, MemoryValidation.instance(), onMissingKey) - } - def validateCpu(cpu: Option[WomValue], onMissingKey: => ErrorOr[Int Refined Positive]): ErrorOr[Int Refined Positive] = { + def validateCpu(cpu: Option[WomValue], + onMissingKey: => ErrorOr[Int Refined Positive] + ): ErrorOr[Int Refined Positive] = validateWithValidation(cpu, CpuValidation.instance, onMissingKey) - } - def validateMaxRetries(maxRetries: Option[WomValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = { + def validateMaxRetries(maxRetries: Option[WomValue], onMissingKey: => ErrorOr[Int]): ErrorOr[Int] = validateWithValidation(maxRetries, MaxRetriesValidation.instance, onMissingKey) - } private def validateWithValidation[T](valueOption: Option[WomValue], validation: RuntimeAttributesValidation[T], - onMissingValue: => ErrorOr[T]): ErrorOr[T] = { + onMissingValue: => ErrorOr[T] + ): ErrorOr[T] = valueOption match { case Some(value) => validation.validateValue.applyOrElse(value, (_: Any) => validation.invalidValueFailure(value)) case None => onMissingValue } - } - def validateInt(value: WomValue): ErrorOr[Int] = { + def validateInt(value: WomValue): ErrorOr[Int] = WomIntegerType.coerceRawValue(value) match { case scala.util.Success(WomInteger(i)) => i.intValue.validNel case _ => s"Could not coerce ${value.valueString} into an integer".invalidNel } - } - def validateBoolean(value: WomValue): ErrorOr[Boolean] = { + def validateBoolean(value: WomValue): ErrorOr[Boolean] = WomBooleanType.coerceRawValue(value) match { case scala.util.Success(WomBoolean(b)) => b.booleanValue.validNel case _ => s"Could not coerce ${value.valueString} into a boolean".invalidNel } - } - def parseMemoryString(k: String, s: WomString): ErrorOr[MemorySize] = { + def parseMemoryString(k: String, s: WomString): ErrorOr[MemorySize] = InformationValidation.validateString(k, s, allowZero = false) - } def withDefault[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType], - default: WomValue): RuntimeAttributesValidation[ValidatedType] = { + default: WomValue + ): RuntimeAttributesValidation[ValidatedType] = new RuntimeAttributesValidation[ValidatedType] { override def key: String = validation.key @@ -101,10 +96,10 @@ object RuntimeAttributesValidation { override protected def staticDefaultOption = Option(default) } - } def withUsedInCallCaching[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType], - usedInCallCachingValue: Boolean): RuntimeAttributesValidation[ValidatedType] = { + usedInCallCachingValue: Boolean + ): RuntimeAttributesValidation[ValidatedType] = new RuntimeAttributesValidation[ValidatedType] { override def key: String = validation.key @@ -125,10 +120,10 @@ object RuntimeAttributesValidation { override protected def staticDefaultOption = validation.staticDefaultOption } - } - def optional[ValidatedType](validation: RuntimeAttributesValidation[ValidatedType]): - OptionalRuntimeAttributesValidation[ValidatedType] = { + def optional[ValidatedType]( + validation: RuntimeAttributesValidation[ValidatedType] + ): OptionalRuntimeAttributesValidation[ValidatedType] = new OptionalRuntimeAttributesValidation[ValidatedType] { override def key: String = validation.key @@ -149,7 +144,6 @@ object RuntimeAttributesValidation { override protected def staticDefaultOption = validation.staticDefaultOption } - } /** * Returns the value from the attributes, unpacking options, and converting them to string values suitable for @@ -181,9 +175,9 @@ object RuntimeAttributesValidation { * @throws ClassCastException if the validation is called on an optional validation. */ def extract[A](runtimeAttributesValidation: RuntimeAttributesValidation[A], - validatedRuntimeAttributes: ValidatedRuntimeAttributes): A = { + validatedRuntimeAttributes: ValidatedRuntimeAttributes + ): A = extract(runtimeAttributesValidation.key, validatedRuntimeAttributes) - } /** * Returns the value from the attributes matching the key. @@ -192,14 +186,15 @@ object RuntimeAttributesValidation { * @param validatedRuntimeAttributes The values to search. * @return The value matching the key. */ - def extract[A](key: String, - validatedRuntimeAttributes: ValidatedRuntimeAttributes): A = { + def extract[A](key: String, validatedRuntimeAttributes: ValidatedRuntimeAttributes): A = { val value = extractOption(key, validatedRuntimeAttributes) value match { // NOTE: Some(innerValue) aka Some.unapply() throws a `ClassCastException` to `Nothing$` as it can't tell the type case some: Some[_] => some.get.asInstanceOf[A] - case None => throw new RuntimeException( - s"$key not found in runtime attributes ${validatedRuntimeAttributes.attributes.keys}") + case None => + throw new RuntimeException( + s"$key not found in runtime attributes ${validatedRuntimeAttributes.attributes.keys}" + ) } } @@ -211,9 +206,9 @@ object RuntimeAttributesValidation { * @return The Some(value) matching the key or None. */ def extractOption[A](runtimeAttributesValidation: RuntimeAttributesValidation[A], - validatedRuntimeAttributes: ValidatedRuntimeAttributes): Option[A] = { + validatedRuntimeAttributes: ValidatedRuntimeAttributes + ): Option[A] = extractOption(runtimeAttributesValidation.key, validatedRuntimeAttributes) - } /** * Returns Some(value) from the attributes matching the key, or None. @@ -234,13 +229,12 @@ object RuntimeAttributesValidation { * @tparam A The type to cast the unpacked value. * @return The Some(value) matching the key or None. */ - final def unpackOption[A](value: Any): Option[A] = { + final def unpackOption[A](value: Any): Option[A] = value match { case None => None case Some(innerValue) => unpackOption(innerValue) case _ => Option(value.asInstanceOf[A]) } - } } /** @@ -251,13 +245,13 @@ case class BadDefaultAttribute(badDefaultValue: WomValue) extends WomValue { val womType = WomStringType } - /** * Performs a validation on a runtime attribute and returns some value. * * @tparam ValidatedType The type of the validated value. */ trait RuntimeAttributesValidation[ValidatedType] { + /** * Returns the key of the runtime attribute. * @@ -297,8 +291,8 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return true if the value can be validated. */ - protected def validateExpression: PartialFunction[WomValue, Boolean] = { - case womValue => coercion.exists(_ == womValue.womType) + protected def validateExpression: PartialFunction[WomValue, Boolean] = { case womValue => + coercion.exists(_ == womValue.womType) } /** @@ -322,7 +316,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return Wrapped invalidValueMessage. */ - protected final def invalidValueFailure(value: WomValue): ErrorOr[ValidatedType] = + final protected def invalidValueFailure(value: WomValue): ErrorOr[ValidatedType] = invalidValueMessage(value).invalidNel /** @@ -337,7 +331,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * * @return Wrapped missingValueMessage. */ - protected final lazy val missingValueFailure: ErrorOr[ValidatedType] = missingValueMessage.invalidNel + final protected lazy val missingValueFailure: ErrorOr[ValidatedType] = missingValueMessage.invalidNel /** * Runs this validation on the value matching key. @@ -347,12 +341,11 @@ trait RuntimeAttributesValidation[ValidatedType] { * @param values The full set of values. * @return The error or valid value for this key. */ - def validate(values: Map[String, WomValue]): ErrorOr[ValidatedType] = { + def validate(values: Map[String, WomValue]): ErrorOr[ValidatedType] = values.get(key) match { case Some(value) => validateValue.applyOrElse(value, (_: Any) => invalidValueFailure(value)) case None => validateNone } - } /** * Used during initialization, returning true if the expression __may be__ valid. @@ -371,7 +364,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * @param wdlExpressionMaybe The optional expression. * @return True if the expression may be evaluated. */ - def validateOptionalWomValue(wdlExpressionMaybe: Option[WomValue]): Boolean = { + def validateOptionalWomValue(wdlExpressionMaybe: Option[WomValue]): Boolean = wdlExpressionMaybe match { case None => staticDefaultOption.isDefined || validateNone.isValid case Some(wdlExpression: WdlExpression) => @@ -381,9 +374,8 @@ trait RuntimeAttributesValidation[ValidatedType] { } case Some(womValue) => validateExpression.applyOrElse(womValue, (_: Any) => false) } - } - def validateOptionalWomExpression(womExpressionMaybe: Option[WomExpression]): Boolean = { + def validateOptionalWomExpression(womExpressionMaybe: Option[WomExpression]): Boolean = womExpressionMaybe match { case None => staticDefaultOption.isDefined || validateNone.isValid case Some(womExpression) => @@ -392,7 +384,7 @@ trait RuntimeAttributesValidation[ValidatedType] { case Invalid(_) => true // If we can't evaluate it, we'll let it pass for now... } } - } + /** * Indicates whether this runtime attribute should be used in call caching calculations. * @@ -410,7 +402,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * Returns an optional version of this validation. */ final lazy val optional: OptionalRuntimeAttributesValidation[ValidatedType] = - RuntimeAttributesValidation.optional(this) + RuntimeAttributesValidation.optional(this) /** * Returns a version of this validation with the default value. @@ -432,7 +424,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * @param optionalRuntimeConfig Optional default runtime attributes config of a particular backend. * @return The new version of this validation. */ - final def configDefaultWomValue(optionalRuntimeConfig: Option[Config]): Option[WomValue] = { + final def configDefaultWomValue(optionalRuntimeConfig: Option[Config]): Option[WomValue] = optionalRuntimeConfig collect { case config if config.hasPath(key) => val value = config.getValue(key).unwrapped() @@ -442,13 +434,11 @@ trait RuntimeAttributesValidation[ValidatedType] { BadDefaultAttribute(WomString(value.toString)) } } - } - final def configDefaultValue(optionalRuntimeConfig: Option[Config]): Option[String] = { + final def configDefaultValue(optionalRuntimeConfig: Option[Config]): Option[String] = optionalRuntimeConfig collect { case config if config.hasPath(key) => config.getValue(key).unwrapped().toString } - } /* Methods below provide aliases to expose protected methods to the package. @@ -457,15 +447,15 @@ trait RuntimeAttributesValidation[ValidatedType] { access the protected values, except the `validation` package that uses these back doors. */ - private[validation] final lazy val validateValuePackagePrivate = validateValue + final private[validation] lazy val validateValuePackagePrivate = validateValue - private[validation] final lazy val validateExpressionPackagePrivate = validateExpression + final private[validation] lazy val validateExpressionPackagePrivate = validateExpression - private[validation] final def invalidValueMessagePackagePrivate(value: WomValue) = invalidValueMessage(value) + final private[validation] def invalidValueMessagePackagePrivate(value: WomValue) = invalidValueMessage(value) - private[validation] final lazy val missingValueMessagePackagePrivate = missingValueMessage + final private[validation] lazy val missingValueMessagePackagePrivate = missingValueMessage - private[validation] final lazy val usedInCallCachingPackagePrivate = usedInCallCaching + final private[validation] lazy val usedInCallCachingPackagePrivate = usedInCallCaching } /** @@ -474,6 +464,7 @@ trait RuntimeAttributesValidation[ValidatedType] { * @tparam ValidatedType The type of the validated value. */ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttributesValidation[Option[ValidatedType]] { + /** * Validates the wdl value. * @@ -484,13 +475,12 @@ trait OptionalRuntimeAttributesValidation[ValidatedType] extends RuntimeAttribut */ protected def validateOption: PartialFunction[WomValue, ErrorOr[ValidatedType]] - override final protected lazy val validateValue = new PartialFunction[WomValue, ErrorOr[Option[ValidatedType]]] { + final override protected lazy val validateValue = new PartialFunction[WomValue, ErrorOr[Option[ValidatedType]]] { override def isDefinedAt(womValue: WomValue): Boolean = validateOption.isDefinedAt(womValue) - override def apply(womValue: WomValue): Validated[NonEmptyList[String], Option[ValidatedType]] = { + override def apply(womValue: WomValue): Validated[NonEmptyList[String], Option[ValidatedType]] = validateOption.apply(womValue).map(Option.apply) - } } - override final protected lazy val validateNone: ErrorOr[None.type] = None.validNel[String] + final override protected lazy val validateNone: ErrorOr[None.type] = None.validNel[String] } diff --git a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala index 6e199c4c4fe..d1a6af6f58d 100644 --- a/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/validation/ValidatedRuntimeAttributesBuilder.scala @@ -38,18 +38,14 @@ trait ValidatedRuntimeAttributesBuilder { /** * Returns validators suitable for BackendWorkflowInitializationActor.runtimeAttributeValidators. */ - final lazy val validatorMap: Map[String, Option[WomExpression] => Boolean] = { - validations.map(validation => - validation.key -> validation.validateOptionalWomExpression _ - ).toMap - } + final lazy val validatorMap: Map[String, Option[WomExpression] => Boolean] = + validations.map(validation => validation.key -> validation.validateOptionalWomExpression _).toMap /** * Returns a map of coercions suitable for RuntimeAttributesDefault.workflowOptionsDefault. */ - final lazy val coercionMap: Map[String, Iterable[WomType]] = { + final lazy val coercionMap: Map[String, Iterable[WomType]] = validations.map(validation => validation.key -> validation.coercion).toMap - } def unsupportedKeys(keys: Seq[String]): Seq[String] = keys.diff(validationKeys) @@ -61,11 +57,12 @@ trait ValidatedRuntimeAttributesBuilder { val runtimeAttributesErrorOr: ErrorOr[ValidatedRuntimeAttributes] = validate(attrs) runtimeAttributesErrorOr match { case Valid(runtimeAttributes) => runtimeAttributes - case Invalid(nel) => throw new RuntimeException with MessageAggregation with NoStackTrace { - override def exceptionContext: String = "Runtime attribute validation failed" + case Invalid(nel) => + throw new RuntimeException with MessageAggregation with NoStackTrace { + override def exceptionContext: String = "Runtime attribute validation failed" - override def errorMessages: Iterable[String] = nel.toList - } + override def errorMessages: Iterable[String] = nel.toList + } } } @@ -73,8 +70,8 @@ trait ValidatedRuntimeAttributesBuilder { val listOfKeysToErrorOrAnys: List[(String, ErrorOr[Any])] = validations.map(validation => validation.key -> validation.validate(values)).toList - val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { - case (key, errorOrAny) => errorOrAny map { any => (key, any) } + val listOfErrorOrKeysToAnys: List[ErrorOr[(String, Any)]] = listOfKeysToErrorOrAnys map { case (key, errorOrAny) => + errorOrAny map { any => (key, any) } } import cats.syntax.traverse._ diff --git a/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala b/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala index ec3644674bb..325060aa4f6 100644 --- a/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala +++ b/backend/src/main/scala/cromwell/backend/validation/exception/ValidationAggregatedException.scala @@ -3,4 +3,5 @@ package cromwell.backend.validation.exception import common.exception.MessageAggregation case class ValidationAggregatedException(override val exceptionContext: String, - override val errorMessages: Iterable[String]) extends MessageAggregation + override val errorMessages: Iterable[String] +) extends MessageAggregation diff --git a/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala b/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala index 43af11d8732..c112b8d5b6b 100644 --- a/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/wfs/DefaultWorkflowPathBuilder.scala @@ -2,7 +2,6 @@ package cromwell.backend.wfs import cromwell.core.path.DefaultPathBuilder - object DefaultWorkflowPathBuilder extends WorkflowPathBuilder { override def pathBuilderOption(params: WorkflowFileSystemProviderParams) = Option(DefaultPathBuilder) } diff --git a/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala index bd39ae2c911..c15ac945c12 100644 --- a/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala +++ b/backend/src/main/scala/cromwell/backend/wfs/WorkflowPathBuilder.scala @@ -1,7 +1,7 @@ package cromwell.backend.wfs import com.typesafe.config.Config -import cromwell.backend.io.{WorkflowPathsWithDocker, WorkflowPaths} +import cromwell.backend.io.{WorkflowPaths, WorkflowPathsWithDocker} import cromwell.backend.{BackendConfigurationDescriptor, BackendWorkflowDescriptor} import cromwell.core.WorkflowOptions import cromwell.core.path.PathBuilder @@ -11,14 +11,16 @@ import scala.concurrent.ExecutionContext object WorkflowPathBuilder { def workflowPaths(configurationDescriptor: BackendConfigurationDescriptor, workflowDescriptor: BackendWorkflowDescriptor, - pathBuilders: List[PathBuilder]): WorkflowPaths = { + pathBuilders: List[PathBuilder] + ): WorkflowPaths = new WorkflowPathsWithDocker(workflowDescriptor, configurationDescriptor.backendConfig, pathBuilders) - } } -final case class WorkflowFileSystemProviderParams(fileSystemConfig: Config, globalConfig: Config, +final case class WorkflowFileSystemProviderParams(fileSystemConfig: Config, + globalConfig: Config, workflowOptions: WorkflowOptions, - fileSystemExecutionContext: ExecutionContext) + fileSystemExecutionContext: ExecutionContext +) trait WorkflowPathBuilder { def pathBuilderOption(params: WorkflowFileSystemProviderParams): Option[PathBuilder] diff --git a/backend/src/test/scala/cromwell/backend/BackendSpec.scala b/backend/src/test/scala/cromwell/backend/BackendSpec.scala index 6c4c7a654e5..835abec8eb3 100644 --- a/backend/src/test/scala/cromwell/backend/BackendSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendSpec.scala @@ -3,7 +3,12 @@ package cromwell.backend import _root_.wdl.draft2.model._ import _root_.wdl.transforms.draft2.wdlom2wom.WdlDraft2WomExecutableMakers._ import common.exception.AggregatedException -import cromwell.backend.BackendJobExecutionActor.{BackendJobExecutionResponse, JobFailedNonRetryableResponse, JobFailedRetryableResponse, JobSucceededResponse} +import cromwell.backend.BackendJobExecutionActor.{ + BackendJobExecutionResponse, + JobFailedNonRetryableResponse, + JobFailedRetryableResponse, + JobSucceededResponse +} import cromwell.backend.io.TestWorkflows._ import cromwell.core.callcaching.NoDocker import cromwell.core.labels.Labels @@ -25,18 +30,17 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = scaled(Span(10, Seconds)), interval = Span(500, Millis)) - def testWorkflow(workflow: TestWorkflow, - backend: BackendJobExecutionActor): Unit = { + def testWorkflow(workflow: TestWorkflow, backend: BackendJobExecutionActor): Unit = executeJobAndAssertOutputs(backend, workflow.expectedResponse) - } def buildWorkflowDescriptor(workflowSource: WorkflowSource, inputFileAsJson: Option[String], options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), runtime: String = "", - labels: Labels = Labels.empty): BackendWorkflowDescriptor = { - val wdlNamespace = WdlNamespaceWithWorkflow.load(workflowSource.replaceAll("RUNTIME", runtime), - Seq.empty[Draft2ImportResolver]).get + labels: Labels = Labels.empty + ): BackendWorkflowDescriptor = { + val wdlNamespace = + WdlNamespaceWithWorkflow.load(workflowSource.replaceAll("RUNTIME", runtime), Seq.empty[Draft2ImportResolver]).get val executable = wdlNamespace.toWomExecutable(inputFileAsJson, NoIoFunctionSet, strictValidation = true) match { case Left(errors) => fail(s"Fail to build wom executable: ${errors.toList.mkString(", ")}") case Right(e) => e @@ -45,7 +49,7 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { BackendWorkflowDescriptor( WorkflowId.randomId(), executable.entryPoint, - executable.resolvedExecutableInputs.flatMap({case (port, v) => v.select[WomValue] map { port -> _ }}), + executable.resolvedExecutableInputs.flatMap { case (port, v) => v.select[WomValue] map { port -> _ } }, options, labels, HogGroup("foo"), @@ -55,68 +59,84 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { } def buildWdlWorkflowDescriptor(workflowSource: WorkflowSource, - inputFileAsJson: Option[String] = None, - options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), - runtime: String = "", - labels: Labels = Labels.empty): BackendWorkflowDescriptor = { - + inputFileAsJson: Option[String] = None, + options: WorkflowOptions = WorkflowOptions(JsObject(Map.empty[String, JsValue])), + runtime: String = "", + labels: Labels = Labels.empty + ): BackendWorkflowDescriptor = buildWorkflowDescriptor(workflowSource, inputFileAsJson, options, runtime, labels) - } - def fqnWdlMapToDeclarationMap(m: Map[String, WomValue]): Map[InputDefinition, WomValue] = { - m map { - case (fqn, v) => - val mockDeclaration = RequiredInputDefinition(fqn, v.womType) - mockDeclaration -> v + def fqnWdlMapToDeclarationMap(m: Map[String, WomValue]): Map[InputDefinition, WomValue] = + m map { case (fqn, v) => + val mockDeclaration = RequiredInputDefinition(fqn, v.womType) + mockDeclaration -> v } - } - def fqnMapToDeclarationMap(m: Map[OutputPort, WomValue]): Map[InputDefinition, WomValue] = { - m map { - case (outputPort, womValue) => RequiredInputDefinition(outputPort.name, womValue.womType) -> womValue + def fqnMapToDeclarationMap(m: Map[OutputPort, WomValue]): Map[InputDefinition, WomValue] = + m map { case (outputPort, womValue) => + RequiredInputDefinition(outputPort.name, womValue.womType) -> womValue } - } def jobDescriptorFromSingleCallWorkflow(workflowDescriptor: BackendWorkflowDescriptor, inputs: Map[String, WomValue], options: WorkflowOptions, - runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] + ): BackendJobDescriptor = { + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get val jobKey = BackendJobDescriptorKey(call, None, 1) val inputDeclarations: Map[InputDefinition, WomValue] = call.inputDefinitionMappings.map { - case (inputDef, resolved) => inputDef -> - resolved.select[WomValue].orElse( - resolved.select[WomExpression] - .map( - _.evaluateValue(inputs, NoIoFunctionSet).getOrElse(fail("Can't evaluate input")) + case (inputDef, resolved) => + inputDef -> + resolved + .select[WomValue] + .orElse( + resolved + .select[WomExpression] + .map( + _.evaluateValue(inputs, NoIoFunctionSet).getOrElse(fail("Can't evaluate input")) + ) ) - ).orElse( - resolved.select[OutputPort] flatMap { - case known if workflowDescriptor.knownValues.contains(known) => Option(workflowDescriptor.knownValues(known)) - case hasDefault if hasDefault.graphNode.isInstanceOf[OptionalGraphInputNodeWithDefault] => - Option(hasDefault.graphNode.asInstanceOf[OptionalGraphInputNodeWithDefault].default - .evaluateValue(inputs, NoIoFunctionSet).getOrElse(fail("Can't evaluate input"))) - case _ => None - } - ).getOrElse { - inputs(inputDef.name) - } + .orElse( + resolved.select[OutputPort] flatMap { + case known if workflowDescriptor.knownValues.contains(known) => + Option(workflowDescriptor.knownValues(known)) + case hasDefault if hasDefault.graphNode.isInstanceOf[OptionalGraphInputNodeWithDefault] => + Option( + hasDefault.graphNode + .asInstanceOf[OptionalGraphInputNodeWithDefault] + .default + .evaluateValue(inputs, NoIoFunctionSet) + .getOrElse(fail("Can't evaluate input")) + ) + case _ => None + } + ) + .getOrElse { + inputs(inputDef.name) + } }.toMap - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, Map.empty).getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test - val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) + val evaluatedAttributes = RuntimeAttributeDefinition + .evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, Map.empty) + .getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test + val runtimeAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, None, Map.empty) } def jobDescriptorFromSingleCallWorkflow(wdl: WorkflowSource, options: WorkflowOptions, - runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { + runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] + ): BackendJobDescriptor = { val workflowDescriptor = buildWdlWorkflowDescriptor(wdl) - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get val jobKey = BackendJobDescriptorKey(call, None, 1) val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues) - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations).getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test - val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) + val evaluatedAttributes = RuntimeAttributeDefinition + .evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations) + .getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test + val runtimeAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, None, Map.empty) } @@ -124,25 +144,33 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { runtime: String, attempt: Int, options: WorkflowOptions, - runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition]): BackendJobDescriptor = { + runtimeAttributeDefinitions: Set[RuntimeAttributeDefinition] + ): BackendJobDescriptor = { val workflowDescriptor = buildWdlWorkflowDescriptor(wdl, runtime = runtime) - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get val jobKey = BackendJobDescriptorKey(call, None, attempt) val inputDeclarations = fqnMapToDeclarationMap(workflowDescriptor.knownValues) - val evaluatedAttributes = RuntimeAttributeDefinition.evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations).getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test - val runtimeAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) + val evaluatedAttributes = RuntimeAttributeDefinition + .evaluateRuntimeAttributes(call.callable.runtimeAttributes, NoIoFunctionSet, inputDeclarations) + .getOrElse(fail("Failed to evaluate runtime attributes")) // .get is OK here because this is a test + val runtimeAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, options)(evaluatedAttributes) BackendJobDescriptor(workflowDescriptor, jobKey, runtimeAttributes, inputDeclarations, NoDocker, None, Map.empty) } def assertResponse(executionResponse: BackendJobExecutionResponse, - expectedResponse: BackendJobExecutionResponse): Unit = { + expectedResponse: BackendJobExecutionResponse + ): Unit = { (executionResponse, expectedResponse) match { - case (JobSucceededResponse(_, _, responseOutputs, _, _, _, _), JobSucceededResponse(_, _, expectedOutputs, _, _, _, _)) => + case (JobSucceededResponse(_, _, responseOutputs, _, _, _, _), + JobSucceededResponse(_, _, expectedOutputs, _, _, _, _) + ) => responseOutputs.outputs.size shouldBe expectedOutputs.outputs.size - responseOutputs.outputs foreach { - case (fqn, out) => - val expectedOut = expectedOutputs.outputs.collectFirst({case (p, v) if p.name == fqn.name => v}) - expectedOut.getOrElse(fail(s"Output ${fqn.name} not found in ${expectedOutputs.outputs.map(_._1.name)}")).valueString shouldBe out.valueString + responseOutputs.outputs foreach { case (fqn, out) => + val expectedOut = expectedOutputs.outputs.collectFirst { case (p, v) if p.name == fqn.name => v } + expectedOut + .getOrElse(fail(s"Output ${fqn.name} not found in ${expectedOutputs.outputs.map(_._1.name)}")) + .valueString shouldBe out.valueString } case (JobFailedNonRetryableResponse(_, failure, _), JobFailedNonRetryableResponse(_, expectedFailure, _)) => failure.getClass shouldBe expectedFailure.getClass @@ -157,19 +185,20 @@ trait BackendSpec extends ScalaFutures with Matchers with ScaledTimeSpans { private def concatenateCauseMessages(t: Throwable): String = t match { case null => "" - case ae: AggregatedException => ae.getMessage + " " + ae.throwables.map(innerT => concatenateCauseMessages(innerT.getCause)).mkString("\n") + case ae: AggregatedException => + ae.getMessage + " " + ae.throwables.map(innerT => concatenateCauseMessages(innerT.getCause)).mkString("\n") case other: Throwable => other.getMessage + concatenateCauseMessages(t.getCause) } def executeJobAndAssertOutputs(backend: BackendJobExecutionActor, - expectedResponse: BackendJobExecutionResponse): Unit = { + expectedResponse: BackendJobExecutionResponse + ): Unit = whenReady(backend.execute) { executionResponse => assertResponse(executionResponse, expectedResponse) } - } def firstJobDescriptorKey(workflowDescriptor: BackendWorkflowDescriptor): BackendJobDescriptorKey = { - val call = workflowDescriptor.callable.graph.nodes.collectFirst({ case t: CommandCallNode => t}).get + val call = workflowDescriptor.callable.graph.nodes.collectFirst { case t: CommandCallNode => t }.get BackendJobDescriptorKey(call, None, 1) } } diff --git a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala index 55772d1de90..c4688d759f8 100644 --- a/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/BackendWorkflowInitializationActorSpec.scala @@ -19,23 +19,23 @@ import wom.values._ import scala.concurrent.Future import scala.util.Try - -class BackendWorkflowInitializationActorSpec extends TestKitSuite - with AnyFlatSpecLike with Matchers with TableDrivenPropertyChecks { +class BackendWorkflowInitializationActorSpec + extends TestKitSuite + with AnyFlatSpecLike + with Matchers + with TableDrivenPropertyChecks { behavior of "BackendWorkflowInitializationActorSpec" - val testPredicateBackendWorkflowInitializationActorRef: - TestActorRef[TestPredicateBackendWorkflowInitializationActor] = + val testPredicateBackendWorkflowInitializationActorRef + : TestActorRef[TestPredicateBackendWorkflowInitializationActor] = TestActorRef[TestPredicateBackendWorkflowInitializationActor] - val testPredicateBackendWorkflowInitializationActor: - TestPredicateBackendWorkflowInitializationActor = + val testPredicateBackendWorkflowInitializationActor: TestPredicateBackendWorkflowInitializationActor = testPredicateBackendWorkflowInitializationActorRef.underlyingActor - val testContinueOnReturnCode: Option[WomValue] => Boolean = { + val testContinueOnReturnCode: Option[WomValue] => Boolean = testPredicateBackendWorkflowInitializationActor.continueOnReturnCodePredicate(valueRequired = false) - } val optionalConfig: Option[Config] = Option(TestConfig.optionalRuntimeConfig) @@ -62,22 +62,31 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite "read_int(\"bad file\")" ) + val starRow = Table( + "value", + "*" + ) + val invalidWdlValueRows = Table( "womValue", WomString(""), WomString("z"), - WomFloat(0.0D), + WomFloat(0.0d), WomArray(WomArrayType(WomBooleanType), Seq(WomBoolean(true))), - WomArray(WomArrayType(WomFloatType), Seq(WomFloat(0.0D))) + WomArray(WomArrayType(WomFloatType), Seq(WomFloat(0.0d))) ) forAll(booleanRows) { value => val womValue = WomBoolean(value) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -86,9 +95,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(value)) } @@ -97,7 +110,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WdlExpression.fromString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } @@ -105,9 +120,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomInteger(value) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -116,9 +135,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -127,7 +150,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WdlExpression.fromString(value.toString) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } @@ -135,9 +160,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomArray(WomArrayType(WomIntegerType), Seq(WomInteger(value))) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -146,9 +175,13 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomArray(WomArrayType(WomStringType), Seq(WomString(value.toString))) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.toOption.get should be(ContinueOnReturnCodeSet(Set(value))) } @@ -157,7 +190,9 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WomArray(WomArrayType(WdlExpressionType), Seq(WdlExpression.fromString(value.toString))) val result = false testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } @@ -165,19 +200,41 @@ class BackendWorkflowInitializationActorSpec extends TestKitSuite val womValue = WdlExpression.fromString(expression) val result = true testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) // NOTE: expressions are never valid to validate } + forAll(starRow) { value => + val womValue = WomString(value) + val result = true + testContinueOnReturnCode(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) + val valid = + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + valid.isValid should be(result) + valid.toEither.toOption.get should be(ContinueOnReturnCodeFlag(true)) + } + forAll(invalidWdlValueRows) { womValue => val result = false testContinueOnReturnCode(Option(womValue)) should be(result) - ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be(result) + ContinueOnReturnCodeValidation.default(optionalConfig).validateOptionalWomValue(Option(womValue)) should be( + result + ) val valid = - ContinueOnReturnCodeValidation.default(optionalConfig).validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) + ContinueOnReturnCodeValidation + .default(optionalConfig) + .validate(Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> womValue)) valid.isValid should be(result) valid.toEither.swap.toOption.get.toList should contain theSameElementsAs List( - "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]" + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." ) } @@ -197,17 +254,18 @@ class TestPredicateBackendWorkflowInitializationActor extends BackendWorkflowIni override protected def coerceDefaultRuntimeAttributes(options: WorkflowOptions): Try[Map[String, WomValue]] = throw new UnsupportedOperationException("coerceDefaultRuntimeAttributes") - override def beforeAll(): Future[Option[BackendInitializationData]] = throw new UnsupportedOperationException("beforeAll") + override def beforeAll(): Future[Option[BackendInitializationData]] = throw new UnsupportedOperationException( + "beforeAll" + ) override def validate(): Future[Unit] = throw new UnsupportedOperationException("validate") override protected def workflowDescriptor: BackendWorkflowDescriptor = throw new UnsupportedOperationException("workflowDescriptor") - override protected def configurationDescriptor: BackendConfigurationDescriptor = BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty()) + override protected def configurationDescriptor: BackendConfigurationDescriptor = + BackendConfigurationDescriptor(TestConfig.sampleBackendRuntimeConfig, ConfigFactory.empty()) - override def continueOnReturnCodePredicate(valueRequired: Boolean) - (wdlExpressionMaybe: Option[WomValue]): Boolean = { + override def continueOnReturnCodePredicate(valueRequired: Boolean)(wdlExpressionMaybe: Option[WomValue]): Boolean = super.continueOnReturnCodePredicate(valueRequired)(wdlExpressionMaybe) - } } diff --git a/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala b/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala index ac3c591f5df..57acada06d3 100644 --- a/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala +++ b/backend/src/test/scala/cromwell/backend/MemorySizeSpec.scala @@ -66,7 +66,7 @@ class MemorySizeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers memorySize.to(newUnit) shouldEqual result } } - + it should "round trip" in { List( "2 GB" diff --git a/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala b/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala index 0959f92577b..997fd6f5df5 100644 --- a/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/OutputEvaluatorSpec.scala @@ -24,7 +24,7 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc behavior of "OutputEvaluator" private val FutureTimeout = 20.seconds - final implicit val blockingEc: ExecutionContextExecutor = ExecutionContext.fromExecutor( + implicit final val blockingEc: ExecutionContextExecutor = ExecutionContext.fromExecutor( Executors.newCachedThreadPool() ) @@ -32,50 +32,56 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc private def o1Expression = new WomExpression { override def sourceString: String = "o1" override def inputs: Set[String] = Set("input") - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = Validated.fromOption(inputValues.get("input"), NonEmptyList.one("Can't find a value for 'input'")) - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = throw new UnsupportedOperationException - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = + throw new UnsupportedOperationException + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException } // Depends on a previous output private def o2Expression = new WomExpression { override def sourceString: String = "o2" override def inputs: Set[String] = Set("o1") - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = Validated.fromOption(inputValues.get("o1"), NonEmptyList.one("Can't find a value for 'o1'")) - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = throw new UnsupportedOperationException - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = + throw new UnsupportedOperationException + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = throw new UnsupportedOperationException } private def invalidWomExpression1 = new WomExpression { override def sourceString: String = "invalid1" override def inputs: Set[String] = Set.empty - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = "Invalid expression 1".invalidNel - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = { + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = "Invalid expression 1".invalidNel - } - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = "Invalid expression 1".invalidNel - } } private def invalidWomExpression2 = new WomExpression { override def sourceString: String = "invalid2" override def inputs: Set[String] = Set.empty - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = "Invalid expression 2".invalidNel - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = { + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = "Invalid expression 2".invalidNel - } - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = "Invalid expression 2".invalidNel - } } val exception = new Exception("Expression evaluation exception") @@ -83,15 +89,15 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc private def throwingWomExpression = new WomExpression { override def sourceString: String = "throwing" override def inputs: Set[String] = Set.empty - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { + override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = throw exception - } - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = { + override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = throw exception - } - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { + override def evaluateFiles(inputTypes: Map[String, WomValue], + ioFunctionSet: IoFunctionSet, + coerceTo: WomType + ): ErrorOr[Set[FileEvaluation]] = throw exception - } } val mockInputs: Map[InputDefinition, WomValue] = Map( @@ -99,7 +105,7 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc ) it should "evaluate valid jobs outputs" in { - val mockOutputs = List ( + val mockOutputs = List( OutputDefinition("o1", WomIntegerType, o1Expression), OutputDefinition("o2", WomIntegerType, o2Expression) ) @@ -109,16 +115,19 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc val jobDescriptor = BackendJobDescriptor(null, key, null, mockInputs, null, None, null) Await.result(OutputEvaluator.evaluateOutputs(jobDescriptor, NoIoFunctionSet), FutureTimeout) match { - case ValidJobOutputs(outputs) => outputs shouldBe CallOutputs(Map( - jobDescriptor.taskCall.outputPorts.find(_.name == "o1").get -> WomInteger(5), - jobDescriptor.taskCall.outputPorts.find(_.name == "o2").get -> WomInteger(5) - )) + case ValidJobOutputs(outputs) => + outputs shouldBe CallOutputs( + Map( + jobDescriptor.taskCall.outputPorts.find(_.name == "o1").get -> WomInteger(5), + jobDescriptor.taskCall.outputPorts.find(_.name == "o2").get -> WomInteger(5) + ) + ) case _ => fail("Failed to evaluate outputs") } } it should "return an InvalidJobOutputs if the evaluation returns ErrorOrs" in { - val mockOutputs = List ( + val mockOutputs = List( OutputDefinition("o1", WomIntegerType, o1Expression), OutputDefinition("invalid1", WomIntegerType, invalidWomExpression1), OutputDefinition("invalid2", WomIntegerType, invalidWomExpression2) @@ -129,15 +138,17 @@ class OutputEvaluatorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc val jobDescriptor = BackendJobDescriptor(null, key, null, mockInputs, null, None, null) Await.result(OutputEvaluator.evaluateOutputs(jobDescriptor, NoIoFunctionSet), FutureTimeout) match { - case InvalidJobOutputs(errors) => errors shouldBe NonEmptyList.of( - "Bad output 'invalid1': Invalid expression 1", "Bad output 'invalid2': Invalid expression 2" - ) + case InvalidJobOutputs(errors) => + errors shouldBe NonEmptyList.of( + "Bad output 'invalid1': Invalid expression 1", + "Bad output 'invalid2': Invalid expression 2" + ) case _ => fail("Output evaluation should have failed") } } it should "return an JobOutputsEvaluationException if the evaluation throws an exception" in { - val mockOutputs = List ( + val mockOutputs = List( OutputDefinition("o1", WomIntegerType, o1Expression), OutputDefinition("invalid1", WomIntegerType, throwingWomExpression) ) diff --git a/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala b/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala index 99bec0baebd..91b6ab06617 100644 --- a/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/RuntimeAttributeValidationSpec.scala @@ -19,12 +19,14 @@ class RuntimeAttributeValidationSpec extends AnyFlatSpec with Matchers with Scal val defaultValue = womValue.asWomExpression val validator: Option[WomExpression] => Boolean = _.contains(defaultValue) assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = defaultRuntimeAttributes, - runtimeAttributes = Map.empty, - runtimeAttributeValidators = Map((attributeName, validator)), - ).isValid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = defaultRuntimeAttributes, + runtimeAttributes = Map.empty, + runtimeAttributeValidators = Map((attributeName, validator)) + ) + .isValid ) } @@ -33,12 +35,14 @@ class RuntimeAttributeValidationSpec extends AnyFlatSpec with Matchers with Scal val defaultRuntimeAttributes = Map(attributeName -> womValue) assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = defaultRuntimeAttributes, - runtimeAttributes = Map.empty, - runtimeAttributeValidators = Map((attributeName, (_: Option[WomExpression]) => false)), - ).isInvalid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = defaultRuntimeAttributes, + runtimeAttributes = Map.empty, + runtimeAttributeValidators = Map((attributeName, (_: Option[WomExpression]) => false)) + ) + .isInvalid ) } @@ -49,47 +53,51 @@ class RuntimeAttributeValidationSpec extends AnyFlatSpec with Matchers with Scal val validator: Option[WomExpression] => Boolean = _.contains(runtimeWomExpression) assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = defaultRuntimeAttributes, - runtimeAttributes = runtimeAttributes, - runtimeAttributeValidators = Map((attributeName, validator)), - ).isValid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = defaultRuntimeAttributes, + runtimeAttributes = runtimeAttributes, + runtimeAttributeValidators = Map((attributeName, validator)) + ) + .isValid ) } it should "fail validation if no setting is present but it should be" in forAll { (taskName: String, attributeName: String) => - val validator: Option[WomExpression] => Boolean = { case None => false case Some(x) => throw new RuntimeException(s"expecting the runtime validator to receive a None but got $x") } assert( - BackendWorkflowInitializationActor.validateRuntimeAttributes( - taskName = taskName, - defaultRuntimeAttributes = Map.empty, - runtimeAttributes = Map.empty, - runtimeAttributeValidators = Map((attributeName, validator)), - ).isInvalid + BackendWorkflowInitializationActor + .validateRuntimeAttributes( + taskName = taskName, + defaultRuntimeAttributes = Map.empty, + runtimeAttributes = Map.empty, + runtimeAttributeValidators = Map((attributeName, validator)) + ) + .isInvalid ) } it should "use the taskName and attribute name in correct places for failures" in forAll { (taskName: String, attributeName: String) => - val validator: Option[WomExpression] => Boolean = { case None => false case Some(x) => throw new RuntimeException(s"expecting the runtime validator to receive a None but got $x") } - BackendWorkflowInitializationActor.validateRuntimeAttributes(taskName, Map.empty, Map.empty, Map((attributeName,validator))).fold( - { errors => - val error = errors.toList.head - withClue("attribute name should be set correctly")(error.runtimeAttributeName shouldBe attributeName) - withClue("task name should be set correctly")(error.jobTag shouldBe taskName) - }, - _ => fail("expecting validation to fail!") - ) + BackendWorkflowInitializationActor + .validateRuntimeAttributes(taskName, Map.empty, Map.empty, Map((attributeName, validator))) + .fold( + { errors => + val error = errors.toList.head + withClue("attribute name should be set correctly")(error.runtimeAttributeName shouldBe attributeName) + withClue("task name should be set correctly")(error.jobTag shouldBe taskName) + }, + _ => fail("expecting validation to fail!") + ) } } diff --git a/backend/src/test/scala/cromwell/backend/TestConfig.scala b/backend/src/test/scala/cromwell/backend/TestConfig.scala index 42050dbe323..82e81818d13 100644 --- a/backend/src/test/scala/cromwell/backend/TestConfig.scala +++ b/backend/src/test/scala/cromwell/backend/TestConfig.scala @@ -30,7 +30,8 @@ object TestConfig { lazy val sampleBackendRuntimeConfig = ConfigFactory.parseString(sampleBackendRuntimeConfigString) - lazy val allRuntimeAttrsConfig = ConfigFactory.parseString(allBackendRuntimeAttrsString).getConfig("default-runtime-attributes") + lazy val allRuntimeAttrsConfig = + ConfigFactory.parseString(allBackendRuntimeAttrsString).getConfig("default-runtime-attributes") lazy val optionalRuntimeConfig = sampleBackendRuntimeConfig.getConfig("default-runtime-attributes") diff --git a/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala b/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala index 1a8a101acf8..94a5b00627c 100644 --- a/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/DirectoryFunctionsSpec.scala @@ -18,10 +18,11 @@ class DirectoryFunctionsSpec extends AnyFlatSpec with CromwellTimeoutSpec with M override def copyFile(source: String, destination: String) = throw new UnsupportedOperationException() override def glob(pattern: String) = throw new UnsupportedOperationException() override def size(path: String) = throw new UnsupportedOperationException() - override def readFile(path: String, maxBytes: Option[Int], failOnOverflow: Boolean) = throw new UnsupportedOperationException() + override def readFile(path: String, maxBytes: Option[Int], failOnOverflow: Boolean) = + throw new UnsupportedOperationException() override def pathFunctions = throw new UnsupportedOperationException() override def writeFile(path: String, content: String) = throw new UnsupportedOperationException() - override implicit def ec = throw new UnsupportedOperationException() + implicit override def ec = throw new UnsupportedOperationException() override def createTemporaryDirectory(name: Option[String]) = throw new UnsupportedOperationException() override def asyncIo = throw new UnsupportedOperationException() } @@ -32,13 +33,12 @@ class DirectoryFunctionsSpec extends AnyFlatSpec with CromwellTimeoutSpec with M val innerDir = (rootDir / "innerDir").createDirectories() val link = innerDir / "linkToRootDirInInnerDir" link.symbolicLinkTo(rootDir) - - def listRecursively(path: String)(visited: Vector[String] = Vector.empty): Iterator[String] = { + + def listRecursively(path: String)(visited: Vector[String] = Vector.empty): Iterator[String] = Await.result(functions.listDirectory(path)(visited), Duration.Inf) flatMap { case IoFile(v) => List(v) case IoDirectory(v) => List(v) ++ listRecursively(v)(visited :+ path) } - } listRecursively(rootDir.pathAsString)().toList shouldBe List(innerDir, link).map(_.pathAsString) } diff --git a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala index 286ed796311..21709bfda58 100644 --- a/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/JobPathsSpec.scala @@ -26,7 +26,7 @@ class JobPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wi | } """.stripMargin - val backendConfig = ConfigFactory.parseString(configString) + val backendConfig = ConfigFactory.parseString(configString) val defaultBackendConfigDescriptor = BackendConfigurationDescriptor(backendConfig, TestConfig.globalConfig) "JobPaths" should "provide correct paths for a job" in { @@ -55,8 +55,9 @@ class JobPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers wi fullPath(s"/cromwell-executions/wf_hello/$id/call-hello") jobPaths.callExecutionDockerRoot.pathAsString shouldBe fullPath(s"/cromwell-executions/wf_hello/$id/call-hello/execution") - jobPaths.toDockerPath(DefaultPathBuilder.get( - s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")).pathAsString shouldBe + jobPaths + .toDockerPath(DefaultPathBuilder.get(s"local-cromwell-executions/wf_hello/$id/call-hello/execution/stdout")) + .pathAsString shouldBe fullPath(s"/cromwell-executions/wf_hello/$id/call-hello/execution/stdout") jobPaths.toDockerPath(DefaultPathBuilder.get("/cromwell-executions/dock/path")).pathAsString shouldBe fullPath("/cromwell-executions/dock/path") diff --git a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala index 8e280eadf4e..f956607a573 100644 --- a/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala +++ b/backend/src/test/scala/cromwell/backend/io/TestWorkflows.scala @@ -7,25 +7,26 @@ object TestWorkflows { case class TestWorkflow(workflowDescriptor: BackendWorkflowDescriptor, config: BackendConfigurationDescriptor, - expectedResponse: BackendJobExecutionResponse) + expectedResponse: BackendJobExecutionResponse + ) val HelloWorld = s""" - |task hello { - | String addressee = "you " - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} + |task hello { + | String addressee = "you " + | command { + | echo "Hello $${addressee}!" + | } + | output { + | String salutation = read_string(stdout()) + | } + | + | RUNTIME + |} + | + |workflow wf_hello { + | call hello + |} """.stripMargin val GoodbyeWorld = @@ -46,25 +47,25 @@ object TestWorkflows { val InputFiles = s""" - |task localize { - | File inputFileFromJson - | File inputFileFromCallInputs - | command { - | cat $${inputFileFromJson} - | echo "" - | cat $${inputFileFromCallInputs} - | } - | output { - | Array[String] out = read_lines(stdout()) - | } - | - | RUNTIME - |} - | - |workflow wf_localize { - | File workflowFile - | call localize { input: inputFileFromCallInputs = workflowFile } - |} + |task localize { + | File inputFileFromJson + | File inputFileFromCallInputs + | command { + | cat $${inputFileFromJson} + | echo "" + | cat $${inputFileFromCallInputs} + | } + | output { + | Array[String] out = read_lines(stdout()) + | } + | + | RUNTIME + |} + | + |workflow wf_localize { + | File workflowFile + | call localize { input: inputFileFromCallInputs = workflowFile } + |} """.stripMargin val Sleep20 = @@ -83,25 +84,25 @@ object TestWorkflows { val Scatter = s""" - |task scattering { - | Int intNumber - | command { - | echo $${intNumber} - | } - | output { - | Int out = read_string(stdout()) - | } - |} - | - |workflow wf_scattering { - | Array[Int] numbers = [1, 2, 3] - | scatter (i in numbers) { - | call scattering { input: intNumber = i } - | } - |} + |task scattering { + | Int intNumber + | command { + | echo $${intNumber} + | } + | output { + | Int out = read_string(stdout()) + | } + |} + | + |workflow wf_scattering { + | Array[Int] numbers = [1, 2, 3] + | scatter (i in numbers) { + | call scattering { input: intNumber = i } + | } + |} """.stripMargin - val OutputProcess = { + val OutputProcess = """ |task localize { | File inputFile @@ -121,9 +122,8 @@ object TestWorkflows { | call localize |} """.stripMargin - } - val MissingOutputProcess = { + val MissingOutputProcess = """ |task localize { | command { @@ -137,5 +137,4 @@ object TestWorkflows { | call localize |} """.stripMargin - } } diff --git a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala index cb53276f499..790d42a7887 100644 --- a/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala +++ b/backend/src/test/scala/cromwell/backend/io/WorkflowPathsSpec.scala @@ -16,16 +16,15 @@ class WorkflowPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche def createConfig(values: Map[String, String]): Config = { val config = mock[Config] - values.foreach { - case (key: String, value: String) => - when(config.hasPath(key)).thenReturn(true) - when(config.getString(key)).thenReturn(value) + values.foreach { case (key: String, value: String) => + when(config.hasPath(key)).thenReturn(true) + when(config.getString(key)).thenReturn(value) } config } def rootConfig(root: Option[String], dockerRoot: Option[String]): Config = { - val values: Map[String,String] = root.map("root" -> _).toMap ++ dockerRoot.map("dockerRoot" -> _).toMap + val values: Map[String, String] = root.map("root" -> _).toMap ++ dockerRoot.map("dockerRoot" -> _).toMap createConfig(values) } @@ -82,9 +81,12 @@ class WorkflowPathsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche val expectedDockerRoot = dockerRoot.getOrElse(WorkflowPathsWithDocker.DefaultDockerRoot) workflowPaths.workflowRoot.pathAsString shouldBe - DefaultPathBuilder.get( - s"$expectedRoot/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" - ).toAbsolutePath.pathAsString + DefaultPathBuilder + .get( + s"$expectedRoot/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" + ) + .toAbsolutePath + .pathAsString workflowPaths.dockerWorkflowRoot.pathAsString shouldBe s"$expectedDockerRoot/rootWorkflow/$rootWorkflowId/call-call1/shard-1/attempt-2/subWorkflow/$subWorkflowId" () } diff --git a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala index 0497c8a1ba1..35c57983e6e 100644 --- a/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/StandardValidatedRuntimeAttributesBuilderSpec.scala @@ -14,37 +14,36 @@ import spray.json.{JsArray, JsBoolean, JsNumber, JsObject, JsValue} import wom.RuntimeAttributesKeys._ import wom.values._ -class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers - with MockSugar { +class StandardValidatedRuntimeAttributesBuilderSpec + extends AnyWordSpecLike + with CromwellTimeoutSpec + with Matchers + with MockSugar { val HelloWorld: String = s""" - |task hello { - | String addressee = "you" - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | - | RUNTIME - |} - | - |workflow hello { - | call hello - |} + |task hello { + | String addressee = "you" + | command { + | echo "Hello $${addressee}!" + | } + | output { + | String salutation = read_string(stdout()) + | } + | + | RUNTIME + |} + | + |workflow hello { + | call hello + |} """.stripMargin + val defaultRuntimeAttributes: Map[String, Any] = + Map(DockerKey -> None, FailOnStderrKey -> false, ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) - val defaultRuntimeAttributes: Map[String, Any] = Map( - DockerKey -> None, - FailOnStderrKey -> false, - ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(0))) - - def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]): WorkflowOptions = { + def workflowOptionsWithDefaultRuntimeAttributes(defaults: Map[String, JsValue]): WorkflowOptions = WorkflowOptions(JsObject(Map("default_runtime_attributes" -> JsObject(defaults)))) - } "SharedFileSystemValidatedRuntimeAttributesBuilder" should { "validate when there are no runtime attributes defined" in { @@ -75,8 +74,11 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with var warnings = List.empty[Any] val mockLogger = mock[Logger] mockLogger.warn(anyString).answers((warnings :+= _): Any => Unit) - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - includeDockerSupport = false, logger = mockLogger) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + includeDockerSupport = false, + logger = mockLogger + ) warnings should contain theSameElementsAs List("Unrecognized runtime attribute keys: docker") } @@ -97,46 +99,60 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with var warnings = List.empty[Any] val mockLogger = mock[Logger] mockLogger.warn(anyString).answers((warnings :+= _): Any => Unit) - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - includeDockerSupport = false, logger = mockLogger) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + includeDockerSupport = false, + logger = mockLogger + ) warnings should contain theSameElementsAs List("Unrecognized runtime attribute keys: docker") } "fail to validate an invalid failOnStderr entry" in { val runtimeAttributes = Map("failOnStderr" -> WomString("yes")) - assertRuntimeAttributesFailedCreation(runtimeAttributes, - "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") + assertRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'" + ) } "use workflow options as default if failOnStdErr key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + (FailOnStderrKey -> true) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes(Map(FailOnStderrKey -> JsBoolean(true))) val runtimeAttributes = Map.empty[String, WomValue] - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - workflowOptions = workflowOptions) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + workflowOptions = workflowOptions + ) } "validate a valid continueOnReturnCode entry" in { val runtimeAttributes = Map("continueOnReturnCode" -> WomInteger(1)) - val expectedRuntimeAttributes = defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) + val expectedRuntimeAttributes = + defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1))) assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes) } "fail to validate an invalid continueOnReturnCode entry" in { val runtimeAttributes = Map("continueOnReturnCode" -> WomString("value")) - assertRuntimeAttributesFailedCreation(runtimeAttributes, "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + assertRuntimeAttributesFailedCreation( + runtimeAttributes, + "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." + ) } "use workflow options as default if continueOnReturnCode key is missing" in { val expectedRuntimeAttributes = defaultRuntimeAttributes + (ContinueOnReturnCodeKey -> ContinueOnReturnCodeSet(Set(1, 2))) val workflowOptions = workflowOptionsWithDefaultRuntimeAttributes( - Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2))))) + Map(ContinueOnReturnCodeKey -> JsArray(Vector(JsNumber(1), JsNumber(2)))) + ) val runtimeAttributes = Map.empty[String, WomValue] - assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, expectedRuntimeAttributes, - workflowOptions = workflowOptions) + assertRuntimeAttributesSuccessfulCreation(runtimeAttributes, + expectedRuntimeAttributes, + workflowOptions = workflowOptions + ) } - } val defaultLogger: Logger = LoggerFactory.getLogger(classOf[StandardValidatedRuntimeAttributesBuilderSpec]) @@ -148,44 +164,52 @@ class StandardValidatedRuntimeAttributesBuilderSpec extends AnyWordSpecLike with expectedRuntimeAttributes: Map[String, Any], includeDockerSupport: Boolean = true, workflowOptions: WorkflowOptions = emptyWorkflowOptions, - logger: Logger = defaultLogger): Unit = { + logger: Logger = defaultLogger + ): Unit = { val builder = if (includeDockerSupport) { - StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional) + StandardValidatedRuntimeAttributesBuilder + .default(mockBackendRuntimeConfig) + .withValidation(DockerValidation.optional) } else { StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig) } val runtimeAttributeDefinitions = builder.definitions.toSet - val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ + val addDefaultsToAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ val validatedRuntimeAttributes = builder.build(addDefaultsToAttributes(runtimeAttributes), logger) - val docker = RuntimeAttributesValidation.extractOption( - DockerValidation.instance, validatedRuntimeAttributes) - val failOnStderr = RuntimeAttributesValidation.extract( - FailOnStderrValidation.instance, validatedRuntimeAttributes) - val continueOnReturnCode = RuntimeAttributesValidation.extract( - ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) + val docker = RuntimeAttributesValidation.extractOption(DockerValidation.instance, validatedRuntimeAttributes) + val failOnStderr = RuntimeAttributesValidation.extract(FailOnStderrValidation.instance, validatedRuntimeAttributes) + val continueOnReturnCode = + RuntimeAttributesValidation.extract(ContinueOnReturnCodeValidation.instance, validatedRuntimeAttributes) docker should be(expectedRuntimeAttributes(DockerKey).asInstanceOf[Option[String]]) failOnStderr should be(expectedRuntimeAttributes(FailOnStderrKey).asInstanceOf[Boolean]) continueOnReturnCode should be( - expectedRuntimeAttributes(ContinueOnReturnCodeKey).asInstanceOf[ContinueOnReturnCode]) + expectedRuntimeAttributes(ContinueOnReturnCodeKey) + ) () } - private def assertRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WomValue], exMsg: String, + private def assertRuntimeAttributesFailedCreation(runtimeAttributes: Map[String, WomValue], + exMsg: String, supportsDocker: Boolean = true, workflowOptions: WorkflowOptions = emptyWorkflowOptions, - logger: Logger = defaultLogger): Unit = { + logger: Logger = defaultLogger + ): Unit = { val thrown = the[RuntimeException] thrownBy { val builder = if (supportsDocker) { - StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig).withValidation(DockerValidation.optional) + StandardValidatedRuntimeAttributesBuilder + .default(mockBackendRuntimeConfig) + .withValidation(DockerValidation.optional) } else { StandardValidatedRuntimeAttributesBuilder.default(mockBackendRuntimeConfig) } val runtimeAttributeDefinitions = builder.definitions.toSet - val addDefaultsToAttributes = RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ + val addDefaultsToAttributes = + RuntimeAttributeDefinition.addDefaultsToAttributes(runtimeAttributeDefinitions, workflowOptions) _ builder.build(addDefaultsToAttributes(runtimeAttributes), logger) } diff --git a/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala b/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala index fdfa7f97e04..f50cba5834c 100644 --- a/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/callcaching/CallCachingBlacklistManagerSpec.scala @@ -8,11 +8,10 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class CallCachingBlacklistManagerSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "CallCachingBlacklistManager" - //noinspection RedundantDefaultArgument + // noinspection RedundantDefaultArgument val workflowSourcesNoGrouping = WorkflowSourceFilesWithoutImports( workflowSource = None, workflowUrl = None, diff --git a/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala b/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala index f546dd63630..0b462c74b81 100644 --- a/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/callcaching/RootWorkflowHashCacheActorSpec.scala @@ -13,8 +13,7 @@ import org.scalatest.flatspec.AnyFlatSpecLike import scala.concurrent.duration._ -class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender - with AnyFlatSpecLike { +class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender with AnyFlatSpecLike { private val fakeWorkflowId = WorkflowId.randomId() private val fakeFileName = "fakeFileName" @@ -25,17 +24,24 @@ class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender props = Props(new RootWorkflowFileHashCacheActor(ioActorProbe.ref, fakeWorkflowId) { override lazy val defaultIoTimeout: FiniteDuration = 1.second }), - name = "rootWorkflowFileHashCacheActor-without-timer", + name = "rootWorkflowFileHashCacheActor-without-timer" ) - val ioHashCommandWithContext = IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName)) + val ioHashCommandWithContext = + IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), + FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName) + ) rootWorkflowFileHashCacheActor ! ioHashCommandWithContext - //wait for timeout + // wait for timeout Thread.sleep(2000) EventFilter.info(msgIoAckWithNoRequesters.format(fakeFileName), occurrences = 1).intercept { - ioActorProbe.send(rootWorkflowFileHashCacheActor, ioHashCommandWithContext.fileHashContext -> IoSuccess(ioHashCommandWithContext.ioHashCommand, "Successful result")) + ioActorProbe.send(rootWorkflowFileHashCacheActor, + ioHashCommandWithContext.fileHashContext -> IoSuccess(ioHashCommandWithContext.ioHashCommand, + "Successful result" + ) + ) } } @@ -46,17 +52,32 @@ class RootWorkflowHashCacheActorSpec extends TestKitSuite with ImplicitSender // Effectively disabling automatic timeout firing here. We'll send RequestTimeout ourselves override lazy val defaultIoTimeout: FiniteDuration = 1.hour }), - "rootWorkflowFileHashCacheActor-with-timer", + "rootWorkflowFileHashCacheActor-with-timer" ) - val ioHashCommandWithContext = IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName)) + val ioHashCommandWithContext = + IoHashCommandWithContext(DefaultIoHashCommand(DefaultPathBuilder.build("").get), + FileHashContext(HashKey(checkForHitOrMiss = false, List.empty), fakeFileName) + ) rootWorkflowFileHashCacheActor ! ioHashCommandWithContext val hashVal = "Success" - EventFilter.info(msgTimeoutAfterIoAck.format(s"FileHashSuccess($hashVal)", ioHashCommandWithContext.fileHashContext.file), occurrences = 1).intercept { - ioActorProbe.send(rootWorkflowFileHashCacheActor, (ioHashCommandWithContext.fileHashContext, IoSuccess(ioHashCommandWithContext.ioHashCommand, hashVal))) - Thread.sleep(2000) // wait for actor to put value into cache - ioActorProbe.send(rootWorkflowFileHashCacheActor, RequestTimeout(ioHashCommandWithContext.fileHashContext -> ioHashCommandWithContext.ioHashCommand, rootWorkflowFileHashCacheActor)) - } + EventFilter + .info(msgTimeoutAfterIoAck.format(s"FileHashSuccess($hashVal)", ioHashCommandWithContext.fileHashContext.file), + occurrences = 1 + ) + .intercept { + ioActorProbe.send( + rootWorkflowFileHashCacheActor, + (ioHashCommandWithContext.fileHashContext, IoSuccess(ioHashCommandWithContext.ioHashCommand, hashVal)) + ) + Thread.sleep(2000) // wait for actor to put value into cache + ioActorProbe.send( + rootWorkflowFileHashCacheActor, + RequestTimeout(ioHashCommandWithContext.fileHashContext -> ioHashCommandWithContext.ioHashCommand, + rootWorkflowFileHashCacheActor + ) + ) + } } } diff --git a/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala b/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala index c7463cc50a0..2c7ec3ba726 100644 --- a/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala +++ b/backend/src/test/scala/cromwell/backend/standard/callcaching/StandardFileHashingActorSpec.scala @@ -18,8 +18,12 @@ import scala.concurrent.duration._ import scala.util.control.NoStackTrace import scala.util.{Failure, Try} -class StandardFileHashingActorSpec extends TestKitSuite with ImplicitSender - with AnyFlatSpecLike with Matchers with MockSugar { +class StandardFileHashingActorSpec + extends TestKitSuite + with ImplicitSender + with AnyFlatSpecLike + with Matchers + with MockSugar { behavior of "StandardFileHashingActor" @@ -127,20 +131,21 @@ object StandardFileHashingActorSpec { def defaultParams(): StandardFileHashingActorParams = defaultParams(testing, testing, testing, testing, testing) - def ioActorParams(ioActor: ActorRef): StandardFileHashingActorParams = { - defaultParams(withJobDescriptor = testing, + def ioActorParams(ioActor: ActorRef): StandardFileHashingActorParams = + defaultParams( + withJobDescriptor = testing, withConfigurationDescriptor = testing, withIoActor = ioActor, withServiceRegistryActor = testing, - withBackendInitializationDataOption = testing) - } + withBackendInitializationDataOption = testing + ) def defaultParams(withJobDescriptor: => BackendJobDescriptor, withConfigurationDescriptor: => BackendConfigurationDescriptor, withIoActor: => ActorRef, withServiceRegistryActor: => ActorRef, withBackendInitializationDataOption: => Option[BackendInitializationData] - ): StandardFileHashingActorParams = new StandardFileHashingActorParams { + ): StandardFileHashingActorParams = new StandardFileHashingActorParams { override def jobDescriptor: BackendJobDescriptor = withJobDescriptor @@ -150,10 +155,10 @@ object StandardFileHashingActorSpec { override def serviceRegistryActor: ActorRef = withServiceRegistryActor - override def backendInitializationDataOption: Option[BackendInitializationData] = withBackendInitializationDataOption + override def backendInitializationDataOption: Option[BackendInitializationData] = + withBackendInitializationDataOption override def fileHashCachingActor: Option[ActorRef] = None } } - diff --git a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala index faf835c6343..bd102bc27c1 100644 --- a/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/ContinueOnReturnCodeSpec.scala @@ -9,12 +9,12 @@ import org.scalatest.wordspec.AnyWordSpecLike class ContinueOnReturnCodeSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers with BeforeAndAfterAll { "Checking for return codes" should { "continue on expected return code flags" in { - val flagTests = Table( - ("flag", "returnCode", "expectedContinue"), - (true, 0, true), - (true, 1, true), - (false, 0, true), - (false, 1, false)) + val flagTests = Table(("flag", "returnCode", "expectedContinue"), + (true, 0, true), + (true, 1, true), + (false, 0, true), + (false, 1, false) + ) forAll(flagTests) { (flag, returnCode, expectedContinue) => ContinueOnReturnCodeFlag(flag).continueFor(returnCode) should be(expectedContinue) @@ -30,11 +30,20 @@ class ContinueOnReturnCodeSpec extends AnyWordSpecLike with CromwellTimeoutSpec (Set(1), 1, true), (Set(0, 1), 0, true), (Set(0, 1), 1, true), - (Set(0, 1), 2, false)) + (Set(0, 1), 2, false) + ) forAll(setTests) { (set, returnCode, expectedContinue) => ContinueOnReturnCodeSet(set).continueFor(returnCode) should be(expectedContinue) } } + + "continue on expected return code string" in { + val flagTests = Table(("string", "returnCode", "expectedContinue"), ("*", 0, true), ("*", 1, true)) + + forAll(flagTests) { (flag, returnCode, expectedContinue) => + ContinueOnReturnCodeFlag(flag == "*").continueFor(returnCode) should be(expectedContinue) + } + } } } diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala index 766384b46fd..6123ecfe53b 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesDefaultSpec.scala @@ -21,9 +21,7 @@ class RuntimeAttributesDefaultSpec extends AnyFlatSpec with CromwellTimeoutSpec ) it should "coerce workflow options from Json to WdlValues" in { - val workflowOptions = WorkflowOptions(JsObject( - "default_runtime_attributes" -> JsObject(map)) - ) + val workflowOptions = WorkflowOptions(JsObject("default_runtime_attributes" -> JsObject(map))) val coercionMap: Map[String, Set[WomType]] = Map( "str" -> Set(WomStringType), @@ -43,9 +41,7 @@ class RuntimeAttributesDefaultSpec extends AnyFlatSpec with CromwellTimeoutSpec } it should "only return default values if they're in the coercionMap" in { - val workflowOptions = WorkflowOptions(JsObject( - "default_runtime_attributes" -> JsObject(map)) - ) + val workflowOptions = WorkflowOptions(JsObject("default_runtime_attributes" -> JsObject(map))) val coercionMap: Map[String, Set[WomType]] = Map( "str" -> Set(WomStringType), @@ -69,9 +65,7 @@ class RuntimeAttributesDefaultSpec extends AnyFlatSpec with CromwellTimeoutSpec } it should "throw an exception if a value can't be coerced" in { - val workflowOptions = WorkflowOptions(JsObject( - "default_runtime_attributes" -> JsObject(map)) - ) + val workflowOptions = WorkflowOptions(JsObject("default_runtime_attributes" -> JsObject(map))) val coercionMap: Map[String, Set[WomType]] = Map( "str" -> Set(WomBooleanType), diff --git a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala index 1752da9014b..dafca35e1b6 100644 --- a/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala +++ b/backend/src/test/scala/cromwell/backend/validation/RuntimeAttributesValidationSpec.scala @@ -12,15 +12,21 @@ import wom.RuntimeAttributesKeys import wom.types._ import wom.values._ -class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeoutSpec with Matchers with BeforeAndAfterAll { +class RuntimeAttributesValidationSpec + extends AnyWordSpecLike + with CromwellTimeoutSpec + with Matchers + with BeforeAndAfterAll { val mockBackendRuntimeConfig = TestConfig.allRuntimeAttrsConfig "RuntimeAttributesValidation" should { "return success when tries to validate a valid Docker entry" in { val dockerValue = Some(WomString("someImage")) - val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateDocker( + dockerValue, + "Failed to get Docker mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.get == "someImage") case Invalid(e) => fail(e.toList.mkString(" ")) @@ -38,8 +44,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure (based on defined HoF) when tries to validate a docker entry but it does not contain a value" in { val dockerValue = None - val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateDocker( + dockerValue, + "Failed to get Docker mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Failed to get Docker mandatory key from runtime attributes") @@ -48,8 +56,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when there is an invalid docker runtime attribute defined" in { val dockerValue = Some(WomInteger(1)) - val result = RuntimeAttributesValidation.validateDocker(dockerValue, - "Failed to get Docker mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateDocker( + dockerValue, + "Failed to get Docker mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting docker runtime attribute to be a String") @@ -58,8 +68,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a failOnStderr boolean entry" in { val failOnStderrValue = Some(WomBoolean(true)) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -68,8 +80,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a failOnStderr 'true' string entry" in { val failOnStderrValue = Some(WomString("true")) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -78,8 +92,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a failOnStderr 'false' string entry" in { val failOnStderrValue = Some(WomString("false")) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(!x) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -88,11 +104,16 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when there is an invalid failOnStderr runtime attribute defined" in { val failOnStderrValue = Some(WomInteger(1)) - val result = RuntimeAttributesValidation.validateFailOnStderr(failOnStderrValue, - "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateFailOnStderr( + failOnStderrValue, + "Failed to get failOnStderr mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'") + case Invalid(e) => + assert( + e.head == "Expecting failOnStderr runtime attribute to be a Boolean or a String with values of 'true' or 'false'" + ) } } @@ -107,8 +128,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode boolean entry" in { val continueOnReturnCodeValue = Some(WomBoolean(true)) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -117,8 +140,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode 'true' string entry" in { val continueOnReturnCodeValue = Some(WomString("true")) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -127,8 +152,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode 'false' string entry" in { val continueOnReturnCodeValue = Some(WomString("false")) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -137,8 +164,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a continueOnReturnCode int entry" in { val continueOnReturnCodeValue = Some(WomInteger(12)) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(12))) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -147,19 +176,26 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when there is an invalid continueOnReturnCode runtime attribute defined" in { val continueOnReturnCodeValue = Some(WomString("yes")) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => - assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + assert( + e.head == "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." + ) } } "return success when there is a valid integer array in continueOnReturnCode runtime attribute" in { val continueOnReturnCodeValue = Some(WomArray(WomArrayType(WomIntegerType), Seq(WomInteger(1), WomInteger(2)))) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeSet(Set(1, 2))) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -167,29 +203,52 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo } "return failure when there is an invalid array in continueOnReturnCode runtime attribute" in { - val continueOnReturnCodeValue = Some(WomArray(WomArrayType(WomStringType), Seq(WomString("one"), WomString("two")))) - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, - "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel) + val continueOnReturnCodeValue = + Some(WomArray(WomArrayType(WomStringType), Seq(WomString("one"), WomString("two")))) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + continueOnReturnCodeValue, + "Failed to get continueOnReturnCode mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting continueOnReturnCode runtime attribute to be either a Boolean, a String 'true' or 'false', or an Array[Int]") + case Invalid(e) => + assert( + e.head == "Expecting returnCodes/continueOnReturnCode" + + " runtime attribute to be either a String '*', 'true', or 'false', a Boolean, or an Array[Int]." + ) } } "return success (based on defined HoF) when tries to validate a continueOnReturnCode entry but it does not contain a value" in { val continueOnReturnCodeValue = None - val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, ContinueOnReturnCodeFlag(false).validNel) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode(continueOnReturnCodeValue, + ContinueOnReturnCodeFlag(false).validNel + ) result match { case Valid(x) => assert(x == ContinueOnReturnCodeFlag(false)) case Invalid(e) => fail(e.toList.mkString(" ")) } } + "return success when tries to validate a valid returnCodes string entry" in { + val returnCodesValue = Some(WomString("*")) + val result = RuntimeAttributesValidation.validateContinueOnReturnCode( + returnCodesValue, + "Failed to get return code mandatory key from runtime attributes".invalidNel + ) + result match { + case Valid(x) => assert(x == ContinueOnReturnCodeFlag(true)) + case Invalid(e) => fail(e.toList.mkString(" ")) + } + } + "return success when tries to validate a valid Integer memory entry" in { val expectedGb = 1 val memoryValue = Some(WomInteger(1 << 30)) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.amount == expectedGb) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -198,8 +257,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid Integer memory entry" in { val memoryValue = Some(WomInteger(-1)) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got -1") @@ -209,8 +270,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a valid String memory entry" in { val expectedGb = 2 val memoryValue = Some(WomString("2 GB")) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.amount == expectedGb) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -219,8 +282,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid size in String memory entry" in { val memoryValue = Some(WomString("0 GB")) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting memory runtime attribute value greater than 0 but got 0.0") @@ -229,28 +294,40 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid String memory entry" in { val memoryValue = Some(WomString("value")) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB") + case Invalid(e) => + assert( + e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: value should be of the form 'X Unit' where X is a number, e.g. 8 GB" + ) } } "return failure when tries to validate an invalid memory entry" in { val memoryValue = Some(WomBoolean(true)) - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") - case Invalid(e) => assert(e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value") + case Invalid(e) => + assert( + e.head == "Expecting memory runtime attribute to be an Integer or String with format '8 GB'. Exception: Not supported WDL type value" + ) } } "return failure when tries to validate a non-provided memory entry" in { val memoryValue = None - val result = RuntimeAttributesValidation.validateMemory(memoryValue, - "Failed to get memory mandatory key from runtime attributes".invalidNel) + val result = RuntimeAttributesValidation.validateMemory( + memoryValue, + "Failed to get memory mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Failed to get memory mandatory key from runtime attributes") @@ -259,8 +336,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return success when tries to validate a valid cpu entry" in { val cpuValue = Some(WomInteger(1)) - val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateCpu(cpuValue, + "Failed to get cpu mandatory key from runtime attributes".invalidNel + ) result match { case Valid(x) => assert(x.value == 1) case Invalid(e) => fail(e.toList.mkString(" ")) @@ -269,8 +348,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate an invalid cpu entry" in { val cpuValue = Some(WomInteger(-1)) - val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateCpu(cpuValue, + "Failed to get cpu mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting cpu runtime attribute value greater than 0") @@ -279,8 +360,10 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo "return failure when tries to validate a non-provided cpu entry" in { val cpuValue = None - val result = RuntimeAttributesValidation.validateCpu(cpuValue, - "Failed to get cpu mandatory key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateCpu(cpuValue, + "Failed to get cpu mandatory key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Failed to get cpu mandatory key from runtime attributes") @@ -306,26 +389,25 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo } "return default values as BadDefaultAttribute when they can't be coerced to expected WdlTypes" in { - val optionalInvalidAttrsConfig = Option(ConfigFactory.parseString( - """ - |cpu = 1.4 - |failOnStderr = "notReal" - |continueOnReturnCode = 0 + val optionalInvalidAttrsConfig = Option(ConfigFactory.parseString(""" + |cpu = 1.4 + |failOnStderr = "notReal" + |continueOnReturnCode = 0 """.stripMargin)) - val defaultVals = Map( - "cpu" -> CpuValidation.configDefaultWomValue(optionalInvalidAttrsConfig).get, - "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get, - "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get - ) + val defaultVals = Map( + "cpu" -> CpuValidation.configDefaultWomValue(optionalInvalidAttrsConfig).get, + "failOnStderr" -> FailOnStderrValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get, + "continueOnReturnCode" -> ContinueOnReturnCodeValidation.configDefaultWdlValue(optionalInvalidAttrsConfig).get + ) - val expectedDefaultVals = Map( - "cpu" -> BadDefaultAttribute(WomString("1.4")), - "failOnStderr" -> BadDefaultAttribute(WomString("notReal")), - "continueOnReturnCode" -> WomInteger(0) - ) + val expectedDefaultVals = Map( + "cpu" -> BadDefaultAttribute(WomString("1.4")), + "failOnStderr" -> BadDefaultAttribute(WomString("notReal")), + "continueOnReturnCode" -> WomInteger(0) + ) - defaultVals shouldBe expectedDefaultVals + defaultVals shouldBe expectedDefaultVals } "should parse memory successfully" in { @@ -338,14 +420,14 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo | } |""".stripMargin - val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") + val backendConfig: Config = + ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig)) - val memoryMinVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMinKey, Some(backendConfig)) - val memoryMaxVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryMaxKey, Some(backendConfig)) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WomLong(2147483648L))) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryMinKey, memoryMinVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WomLong(322122547L))) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryMaxKey, memoryMaxVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some((WomLong(429496729L))) + MemoryValidation + .withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get) + .runtimeAttributeDefinition + .factoryDefault shouldBe Some(WomLong(2147483648L)) } "shouldn't throw up if the value for a default-runtime-attribute key cannot be coerced into an expected WomType" in { @@ -356,25 +438,33 @@ class RuntimeAttributesValidationSpec extends AnyWordSpecLike with CromwellTimeo | } |""".stripMargin - val backendConfig: Config = ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") + val backendConfig: Config = + ConfigFactory.parseString(backendConfigTemplate).getConfig("default-runtime-attributes") val memoryVal = MemoryValidation.configDefaultString(RuntimeAttributesKeys.MemoryKey, Some(backendConfig)) - MemoryValidation.withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get).runtimeAttributeDefinition.factoryDefault shouldBe Some(BadDefaultAttribute(WomString("blahblah"))) + MemoryValidation + .withDefaultMemory(RuntimeAttributesKeys.MemoryKey, memoryVal.get) + .runtimeAttributeDefinition + .factoryDefault shouldBe Some(BadDefaultAttribute(WomString("blahblah"))) } "should be able to coerce a list of return codes into an WdlArray" in { - val optinalBackendConfig = Option(ConfigFactory.parseString( - s""" - |continueOnReturnCode = [0,1,2] - |""".stripMargin)) + val optinalBackendConfig = Option(ConfigFactory.parseString(s""" + |continueOnReturnCode = [0,1,2] + |""".stripMargin)) - ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1), WomInteger(2))) + ContinueOnReturnCodeValidation.configDefaultWdlValue(optinalBackendConfig).get shouldBe WomArray( + WomArrayType(WomIntegerType), + List(WomInteger(0), WomInteger(1), WomInteger(2)) + ) } "return failure when tries to validate an invalid maxRetries entry" in { val maxRetries = Option(WomInteger(-1)) - val result = RuntimeAttributesValidation.validateMaxRetries(maxRetries, - "Failed to get maxRetries key from runtime attributes".invalidNel) + val result = + RuntimeAttributesValidation.validateMaxRetries(maxRetries, + "Failed to get maxRetries key from runtime attributes".invalidNel + ) result match { case Valid(_) => fail("A failure was expected.") case Invalid(e) => assert(e.head == "Expecting maxRetries runtime attribute value greater than or equal to 0") diff --git a/build.sbt b/build.sbt index 3b255d0e0e5..8e480907c5e 100644 --- a/build.sbt +++ b/build.sbt @@ -37,6 +37,11 @@ lazy val wdlModelBiscayne = (project in wdlModelRoot / "biscayne") .dependsOn(wdlModelDraft3) .dependsOn(common % "test->test") +lazy val wdlModelCascades = (project in wdlModelRoot / "cascades") + .withLibrarySettings("cromwell-wdl-model-cascades") + .dependsOn(wdlSharedModel) + .dependsOn(common % "test->test") + lazy val wdlTransformsRoot = wdlRoot / "transforms" lazy val wdlSharedTransforms = (project in wdlTransformsRoot / "shared") @@ -74,11 +79,12 @@ lazy val wdlTransformsBiscayne = (project in wdlTransformsRoot / "biscayne") .dependsOn(common % "test->test") .dependsOn(wom % "test->test") -lazy val cwl = project - .withLibrarySettings("cromwell-cwl", cwlDependencies) - .dependsOn(wom) - .dependsOn(wom % "test->test") +lazy val wdlTransformsCascades = (project in wdlTransformsRoot / "cascades") + .withLibrarySettings("cromwell-wdl-transforms-cascades", wdlDependencies) + .dependsOn(wdlNewBaseTransforms) + .dependsOn(languageFactoryCore) .dependsOn(common % "test->test") + .dependsOn(wom % "test->test") lazy val core = project .withLibrarySettings("cromwell-core", coreDependencies) @@ -91,11 +97,17 @@ lazy val cloudSupport = project .dependsOn(common) .dependsOn(common % "test->test") +lazy val azureBlobNio = (project in file("azure-blob-nio")) + .withLibrarySettings("cromwell-azure-blobNio", azureBlobNioDependencies) + lazy val azureBlobFileSystem = (project in file("filesystems/blob")) .withLibrarySettings("cromwell-azure-blobFileSystem", blobFileSystemDependencies) .dependsOn(core) + .dependsOn(cloudSupport) + .dependsOn(azureBlobNio) .dependsOn(core % "test->test") .dependsOn(common % "test->test") + .dependsOn(azureBlobNio % "test->test") lazy val awsS3FileSystem = (project in file("filesystems/s3")) .withLibrarySettings("cromwell-aws-s3filesystem", s3FileSystemDependencies) @@ -154,6 +166,7 @@ lazy val databaseMigration = (project in file("database/migration")) lazy val dockerHashing = project .withLibrarySettings("cromwell-docker-hashing", dockerHashingDependencies) + .dependsOn(cloudSupport) .dependsOn(core) .dependsOn(core % "test->test") .dependsOn(common % "test->test") @@ -183,7 +196,7 @@ lazy val services = project .dependsOn(wdlDraft2LanguageFactory % "test->test") // because the WaaS tests init language config with all languages .dependsOn(wdlDraft3LanguageFactory % "test->test") .dependsOn(wdlBiscayneLanguageFactory % "test->test") - .dependsOn(cwlV1_0LanguageFactory % "test->test") + .dependsOn(wdlCascadesLanguageFactory % "test->test") .dependsOn(core % "test->test") .dependsOn(ftpFileSystem % "test->test") .dependsOn(common % "test->test") @@ -222,6 +235,19 @@ lazy val googlePipelinesV2Beta = (project in backendRoot / "google" / "pipelines .dependsOn(core % "test->test") .dependsOn(common % "test->test") +lazy val googleBatch = (project in backendRoot / "google" / "batch") + .withLibrarySettings("cromwell-google-batch-backend") + .dependsOn(backend) + .dependsOn(gcsFileSystem) + .dependsOn(drsFileSystem) + .dependsOn(sraFileSystem) + .dependsOn(httpFileSystem) + .dependsOn(backend % "test->test") + .dependsOn(gcsFileSystem % "test->test") + .dependsOn(services % "test->test") + .dependsOn(common % "test->test") + .dependsOn(core % "test->test") + lazy val awsBackend = (project in backendRoot / "aws") .withLibrarySettings("cromwell-aws-backend") .dependsOn(backend) @@ -245,6 +271,9 @@ lazy val tesBackend = (project in backendRoot / "tes") .dependsOn(sfsBackend) .dependsOn(ftpFileSystem) .dependsOn(drsFileSystem) + .dependsOn(azureBlobFileSystem) + // TES backend provides a compatibility layer to run WDLs with PAPI runtime attributes [WX-769] + .dependsOn(googlePipelinesCommon) .dependsOn(backend % "test->test") .dependsOn(common % "test->test") @@ -253,6 +282,7 @@ lazy val engine = project .dependsOn(backend) .dependsOn(gcsFileSystem) .dependsOn(drsFileSystem) + .dependsOn(httpFileSystem) .dependsOn(sraFileSystem) .dependsOn(awsS3FileSystem) .dependsOn(azureBlobFileSystem) @@ -263,10 +293,10 @@ lazy val engine = project .dependsOn(azureBlobFileSystem % "test->test") .dependsOn(`cloud-nio-spi`) .dependsOn(languageFactoryCore) - .dependsOn(cwlV1_0LanguageFactory % "test->test") .dependsOn(wdlDraft2LanguageFactory % "test->test") .dependsOn(wdlDraft3LanguageFactory % "test->test") .dependsOn(wdlBiscayneLanguageFactory % "test->test") + .dependsOn(wdlCascadesLanguageFactory % "test->test") .dependsOn(common % "test->test") .dependsOn(core % "test->test") .dependsOn(backend % "test->test") @@ -278,20 +308,12 @@ lazy val engine = project // Executables -lazy val centaurCwlRunner = project - .withExecutableSettings("centaur-cwl-runner", centaurCwlRunnerDependencies, buildDocker = false) - .dependsOn(cwl) - .dependsOn(centaur) - .dependsOn(gcsFileSystem) - .dependsOn(ftpFileSystem) - .dependsOn(common % "test->test") - lazy val womtool = project .withExecutableSettings("womtool", womtoolDependencies) .dependsOn(wdlDraft2LanguageFactory) .dependsOn(wdlDraft3LanguageFactory) .dependsOn(wdlBiscayneLanguageFactory) - .dependsOn(cwlV1_0LanguageFactory) + .dependsOn(wdlCascadesLanguageFactory) .dependsOn(wom % "test->test") .dependsOn(common % "test->test") @@ -337,10 +359,11 @@ lazy val wdlBiscayneLanguageFactory = (project in languageFactoryRoot / "wdl-bis .dependsOn(wdlTransformsBiscayne) .dependsOn(common % "test->test") -lazy val cwlV1_0LanguageFactory = (project in languageFactoryRoot / "cwl-v1-0") - .withLibrarySettings("cwl-v1-0") +lazy val wdlCascadesLanguageFactory = (project in languageFactoryRoot / "wdl-cascades") + .withLibrarySettings("wdl-cascades") .dependsOn(languageFactoryCore) - .dependsOn(cwl) + .dependsOn(wdlModelCascades) + .dependsOn(wdlTransformsCascades) .dependsOn(common % "test->test") lazy val `cloud-nio-spi` = (project in cloudNio / "cloud-nio-spi") @@ -360,11 +383,7 @@ lazy val `cloud-nio-impl-ftp` = (project in cloudNio / "cloud-nio-impl-ftp") lazy val `cloud-nio-impl-drs` = (project in cloudNio / "cloud-nio-impl-drs") .withLibrarySettings(libraryName = "cloud-nio-impl-drs", dependencies = implDrsDependencies) .dependsOn(`cloud-nio-util`) - .dependsOn(common) - .dependsOn(common % "test->test") - -lazy val perf = project - .withExecutableSettings("perf", dependencies = perfDependencies, pushDocker = false) + .dependsOn(cloudSupport) .dependsOn(common) .dependsOn(common % "test->test") @@ -374,18 +393,26 @@ lazy val `cromwell-drs-localizer` = project .dependsOn(common) .dependsOn(`cloud-nio-impl-drs` % "test->test") +lazy val pact4s = project.in(file("pact4s")) + .settings(pact4sSettings) + .dependsOn(engine) + .dependsOn(services) + .dependsOn(engine % "test->test") + .disablePlugins(sbtassembly.AssemblyPlugin) + lazy val server = project .withExecutableSettings("cromwell", serverDependencies) .dependsOn(engine) .dependsOn(googlePipelinesV2Alpha1) .dependsOn(googlePipelinesV2Beta) + .dependsOn(googleBatch) .dependsOn(awsBackend) .dependsOn(tesBackend) .dependsOn(cromwellApiClient) .dependsOn(wdlDraft2LanguageFactory) .dependsOn(wdlDraft3LanguageFactory) .dependsOn(wdlBiscayneLanguageFactory) - .dependsOn(cwlV1_0LanguageFactory) + .dependsOn(wdlCascadesLanguageFactory) .dependsOn(engine % "test->test") .dependsOn(common % "test->test") @@ -399,17 +426,15 @@ lazy val root = (project in file(".")) .aggregate(`cromwell-drs-localizer`) .aggregate(awsBackend) .aggregate(awsS3FileSystem) + .aggregate(azureBlobNio) .aggregate(azureBlobFileSystem) .aggregate(backend) .aggregate(centaur) - .aggregate(centaurCwlRunner) .aggregate(cloudSupport) .aggregate(common) .aggregate(core) .aggregate(cromiam) .aggregate(cromwellApiClient) - .aggregate(cwl) - .aggregate(cwlV1_0LanguageFactory) .aggregate(databaseMigration) .aggregate(databaseSql) .aggregate(dockerHashing) @@ -420,27 +445,31 @@ lazy val root = (project in file(".")) .aggregate(googlePipelinesCommon) .aggregate(googlePipelinesV2Alpha1) .aggregate(googlePipelinesV2Beta) + .aggregate(googleBatch) .aggregate(httpFileSystem) .aggregate(languageFactoryCore) - .aggregate(perf) .aggregate(server) .aggregate(services) .aggregate(sfsBackend) .aggregate(sraFileSystem) .aggregate(tesBackend) .aggregate(wdlBiscayneLanguageFactory) + .aggregate(wdlCascadesLanguageFactory) .aggregate(wdlDraft2LanguageFactory) .aggregate(wdlDraft3LanguageFactory) .aggregate(wdlModelBiscayne) + .aggregate(wdlModelCascades) .aggregate(wdlModelDraft2) .aggregate(wdlModelDraft3) .aggregate(wdlNewBaseTransforms) .aggregate(wdlSharedModel) .aggregate(wdlSharedTransforms) .aggregate(wdlTransformsBiscayne) + .aggregate(wdlTransformsCascades) .aggregate(wdlTransformsDraft2) .aggregate(wdlTransformsDraft3) .aggregate(wes2cromwell) .aggregate(wom) .aggregate(womtool) + .aggregate(pact4s) .withAggregateSettings() diff --git a/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala b/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala index 1ddebb6a09a..f90fb21e537 100644 --- a/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/AbstractCentaurTestCaseSpec.scala @@ -20,7 +20,10 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.Future @DoNotDiscover -abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromwellTracker: Option[CromwellTracker] = None) extends AsyncFlatSpec with Matchers { +abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], + cromwellTracker: Option[CromwellTracker] = None +) extends AsyncFlatSpec + with Matchers { /* NOTE: We need to statically initialize the object so that the exceptions appear here in the class constructor. @@ -47,10 +50,12 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw val duplicateTestNames = allTestsCases .map(_.workflow.testName) .groupBy(identity) - .collect({ case (key, values) if values.lengthCompare(1) > 0 => key }) + .collect { case (key, values) if values.lengthCompare(1) > 0 => key } if (duplicateTestNames.nonEmpty) { - throw new RuntimeException("The following test names are duplicated in more than one test file: " + - duplicateTestNames.mkString(", ")) + throw new RuntimeException( + "The following test names are duplicated in more than one test file: " + + duplicateTestNames.mkString(", ") + ) } allTestsCases } @@ -62,75 +67,28 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw } yield submitResponse // Make tags, but enforce lowercase: - val tags = (testCase.testOptions.tags :+ testCase.workflow.testName :+ testCase.testFormat.name) map { x => Tag(x.toLowerCase) } + val tags = (testCase.testOptions.tags :+ testCase.workflow.testName :+ testCase.testFormat.name) map { x => + Tag(x.toLowerCase) + } val isIgnored = testCase.isIgnored(cromwellBackends) val retries = if (testCase.workflow.retryTestFailures) ErrorReporters.retryAttempts else 0 runOrDont(testCase, tags, isIgnored, retries, runTestAndDeleteZippedImports()) } - def executeWdlUpgradeTest(testCase: CentaurTestCase): Unit = - executeStandardTest(wdlUpgradeTestWdl(testCase)) - - private def wdlUpgradeTestWdl(testCase: CentaurTestCase): CentaurTestCase = { - import better.files.File - import womtool.WomtoolMain - - // The suffix matters because WomGraphMaker.getBundle() uses it to choose the language factory - val rootWorkflowFile = File.newTemporaryFile(suffix = ".wdl").append(testCase.workflow.data.workflowContent.get) - val workingDir = File.newTemporaryDirectory() - val upgradedImportsDir = File.newTemporaryDirectory() - val rootWorkflowFilepath = workingDir / rootWorkflowFile.name - - // Un-upgraded imports go into the working directory - testCase.workflow.data.zippedImports match { - case Some(importsZip: File) => - importsZip.unzipTo(workingDir) - case None => () - } - - // Upgrade the imports and copy to main working dir (precludes transitive imports; no recursion yet) - workingDir.list.toList.map { file: File => - val upgradedWdl = WomtoolMain.upgrade(file.pathAsString).stdout.get - upgradedImportsDir.createChild(file.name).append(upgradedWdl) - } - - // Copy to working directory after we operate on the imports that are in it - rootWorkflowFile.copyTo(rootWorkflowFilepath) - - val upgradeResult = WomtoolMain.upgrade(rootWorkflowFilepath.pathAsString) - - upgradeResult.stderr match { - case Some(stderr) => println(stderr) - case _ => () - } - - val newCase = testCase.copy( - workflow = testCase.workflow.copy( - testName = testCase.workflow.testName + " (draft-2 to 1.0 upgrade)", - data = testCase.workflow.data.copy( - workflowContent = Option(upgradeResult.stdout.get), // this '.get' catches an error if upgrade fails - zippedImports = Option(upgradedImportsDir.zip()))))(cromwellTracker) // An empty zip appears to be completely harmless, so no special handling - - rootWorkflowFile.delete(swallowIOExceptions = true) - upgradedImportsDir.delete(swallowIOExceptions = true) - workingDir.delete(swallowIOExceptions = true) - - newCase - } - private def runOrDont(testCase: CentaurTestCase, tags: List[Tag], ignore: Boolean, retries: Int, - runTest: => IO[SubmitResponse]): Unit = { + runTest: => IO[SubmitResponse] + ): Unit = { val itShould: ItVerbString = it should testCase.name tags match { case Nil => runOrDont(itShould, ignore, testCase, retries, runTest) case head :: Nil => runOrDont(itShould taggedAs head, ignore, testCase, retries, runTest) - case head :: tail => runOrDont(itShould taggedAs(head, tail: _*), ignore, testCase, retries, runTest) + case head :: tail => runOrDont(itShould taggedAs (head, tail: _*), ignore, testCase, retries, runTest) } } @@ -138,26 +96,26 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw ignore: Boolean, testCase: CentaurTestCase, retries: Int, - runTest: => IO[SubmitResponse]): Unit = { + runTest: => IO[SubmitResponse] + ): Unit = if (ignore) { itVerbString ignore Future.successful(succeed) } else { itVerbString in tryTryAgain(testCase, runTest, retries).unsafeToFuture().map(_ => succeed) } - } private def runOrDont(itVerbStringTaggedAs: ItVerbStringTaggedAs, ignore: Boolean, testCase: CentaurTestCase, retries: Int, - runTest: => IO[SubmitResponse]): Unit = { + runTest: => IO[SubmitResponse] + ): Unit = if (ignore) { itVerbStringTaggedAs ignore Future.successful(succeed) } else { itVerbStringTaggedAs in tryTryAgain(testCase, runTest, retries).unsafeToFuture().map(_ => succeed) } - } /** * Returns an IO effect that will recursively try to run a test. @@ -168,21 +126,27 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw * @param attempt Current zero based attempt. * @return IO effect that will run the test, possibly retrying. */ - private def tryTryAgain(testCase: CentaurTestCase, runTest: => IO[SubmitResponse], retries: Int, attempt: Int = 0): IO[SubmitResponse] = { + private def tryTryAgain(testCase: CentaurTestCase, + runTest: => IO[SubmitResponse], + retries: Int, + attempt: Int = 0 + ): IO[SubmitResponse] = { def maybeRetry(centaurTestException: CentaurTestException): IO[SubmitResponse] = { - def clearCachedResults(workflowId: WorkflowId): IO[Unit] = CromwellDatabaseCallCaching.clearCachedResults(workflowId.toString) + def clearCachedResults(workflowId: WorkflowId): IO[Unit] = + CromwellDatabaseCallCaching.clearCachedResults(workflowId.toString) val testEnvironment = TestEnvironment(testCase, retries, attempt) for { _ <- ErrorReporters.logFailure(testEnvironment, centaurTestException) - r <- if (attempt < retries) { - testCase.submittedWorkflowTracker.cleanUpBeforeRetry(clearCachedResults) *> - tryTryAgain(testCase, runTest, retries, attempt + 1) - } else { - IO.raiseError(centaurTestException) - } + r <- + if (attempt < retries) { + testCase.submittedWorkflowTracker.cleanUpBeforeRetry(clearCachedResults) *> + tryTryAgain(testCase, runTest, retries, attempt + 1) + } else { + IO.raiseError(centaurTestException) + } } yield r } @@ -207,11 +171,10 @@ abstract class AbstractCentaurTestCaseSpec(cromwellBackends: List[String], cromw /** * Clean up temporary zip files created for Imports testing. */ - private def cleanUpImports(wfData: WorkflowData) = { + private def cleanUpImports(wfData: WorkflowData) = wfData.zippedImports match { case Some(zipFile) => zipFile.delete(swallowIOExceptions = true) case None => // } - } } diff --git a/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala index c55c0c5b17a..04f03d1eacf 100644 --- a/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/AbstractCromwellEngineOrBackendUpgradeTestCaseSpec.scala @@ -13,7 +13,7 @@ import scala.concurrent.Future @DoNotDiscover abstract class AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCentaurTestCaseSpec(cromwellBackends) + extends AbstractCentaurTestCaseSpec(cromwellBackends) with CentaurTestSuiteShutdown with BeforeAndAfter { @@ -27,15 +27,20 @@ abstract class AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBacken override protected def beforeAll(): Unit = { super.beforeAll() val beforeAllIo = for { - _ <- checkIsEmpty(cromwellDatabase.engineDatabase, cromwellDatabase.engineDatabase.existsJobKeyValueEntries(), testType) - _ <- checkIsEmpty(cromwellDatabase.metadataDatabase, cromwellDatabase.metadataDatabase.existsMetadataEntries(), testType) + _ <- checkIsEmpty(cromwellDatabase.engineDatabase, + cromwellDatabase.engineDatabase.existsJobKeyValueEntries(), + testType + ) + _ <- checkIsEmpty(cromwellDatabase.metadataDatabase, + cromwellDatabase.metadataDatabase.existsMetadataEntries(), + testType + ) } yield () beforeAllIo.unsafeRunSync() } - private def failNotSlick(database: SqlDatabase): IO[Unit] = { + private def failNotSlick(database: SqlDatabase): IO[Unit] = IO.raiseError(new RuntimeException(s"Expected a slick database for ${database.connectionDescription}.")) - } after { val afterIo = for { @@ -54,26 +59,29 @@ abstract class AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBacken def isMatchingUpgradeTest(testCase: CentaurTestCase): Boolean } - object AbstractCromwellEngineOrBackendUpgradeTestCaseSpec { - private def checkIsEmpty(database: SqlDatabase, lookup: => Future[Boolean], testType: => String)(implicit cs: ContextShift[IO]): IO[Unit] = { - IO.fromFuture(IO(lookup)).flatMap(exists => - if (exists) { - IO(Assertions.fail( - s"Database ${database.connectionDescription} contains data. " + - s"$testType tests should only be run on a completely empty database. " + - "You may need to manually drop and recreate the database to continue." - )) - } else { - IO.unit - } - ) - } + private def checkIsEmpty(database: SqlDatabase, lookup: => Future[Boolean], testType: => String)(implicit + cs: ContextShift[IO] + ): IO[Unit] = + IO.fromFuture(IO(lookup)) + .flatMap(exists => + if (exists) { + IO( + Assertions.fail( + s"Database ${database.connectionDescription} contains data. " + + s"$testType tests should only be run on a completely empty database. " + + "You may need to manually drop and recreate the database to continue." + ) + ) + } else { + IO.unit + } + ) private def recreateDatabase(slickDatabase: SlickDatabase)(implicit cs: ContextShift[IO]): IO[Unit] = { import slickDatabase.dataAccess.driver.api._ val schemaName = slickDatabase.databaseConfig.getOrElse("db.cromwell-database-name", "cromwell_test") - //noinspection SqlDialectInspection + // noinspection SqlDialectInspection for { _ <- IO.fromFuture(IO(slickDatabase.database.run(sqlu"""DROP SCHEMA IF EXISTS #$schemaName"""))) _ <- IO.fromFuture(IO(slickDatabase.database.run(sqlu"""CREATE SCHEMA #$schemaName"""))) diff --git a/centaur/src/it/scala/centaur/CentaurTestSuite.scala b/centaur/src/it/scala/centaur/CentaurTestSuite.scala index 1746caee32b..c72d774c4e4 100644 --- a/centaur/src/it/scala/centaur/CentaurTestSuite.scala +++ b/centaur/src/it/scala/centaur/CentaurTestSuite.scala @@ -16,17 +16,17 @@ object CentaurTestSuite extends StrictLogging { // before we can generate the tests. startCromwell() - def startCromwell(): Unit = { + def startCromwell(): Unit = CentaurConfig.runMode match { case ManagedCromwellServer(preRestart, _, _) => CromwellManager.startCromwell(preRestart) case _ => } - } val cromwellBackends = CentaurCromwellClient.backends.unsafeRunSync().supportedBackends.map(_.toLowerCase) - - def isWdlUpgradeTest(testCase: CentaurTestCase): Boolean = testCase.containsTag("wdl_upgrade") + val defaultBackend = CentaurCromwellClient.backends.unsafeRunSync().defaultBackend.toLowerCase + logger.info(s"Cromwell under test configured with backends ${cromwellBackends.mkString(", ")}") + logger.info(s"Unless overridden by workflow options file, tests use default backend: $defaultBackend") def isEngineUpgradeTest(testCase: CentaurTestCase): Boolean = testCase.containsTag("engine_upgrade") @@ -63,9 +63,8 @@ object CentaurTestSuite extends StrictLogging { trait CentaurTestSuiteShutdown extends Suite with BeforeAndAfterAll { private var shutdownHook: Option[ShutdownHookThread] = _ - override protected def beforeAll() = { - shutdownHook = Option(sys.addShutdownHook { CromwellManager.stopCromwell("JVM Shutdown Hook") }) - } + override protected def beforeAll() = + shutdownHook = Option(sys.addShutdownHook(CromwellManager.stopCromwell("JVM Shutdown Hook"))) override protected def afterAll() = { CromwellManager.stopCromwell("ScalaTest AfterAll") @@ -78,6 +77,6 @@ trait CentaurTestSuiteShutdown extends Suite with BeforeAndAfterAll { * The main centaur test suites, runs sub suites in parallel, but allows better control over the way each nested suite runs. */ class CentaurTestSuite - extends Suites(new SequentialTestCaseSpec(), new ParallelTestCaseSpec()) + extends Suites(new SequentialTestCaseSpec(), new ParallelTestCaseSpec()) with ParallelTestExecution with CentaurTestSuiteShutdown diff --git a/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala index a16dda39b49..7a386f9d642 100644 --- a/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/EngineUpgradeTestCaseSpec.scala @@ -4,12 +4,13 @@ import centaur.test.standard.CentaurTestCase import org.scalatest.DoNotDiscover @DoNotDiscover -class EngineUpgradeTestCaseSpec(cromwellBackends: List[String]) extends - AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { +class EngineUpgradeTestCaseSpec(cromwellBackends: List[String]) + extends AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { def this() = this(CentaurTestSuite.cromwellBackends) override def testType: String = "Engine upgrade" - override def isMatchingUpgradeTest(testCase: CentaurTestCase): Boolean = CentaurTestSuite.isEngineUpgradeTest(testCase) + override def isMatchingUpgradeTest(testCase: CentaurTestCase): Boolean = + CentaurTestSuite.isEngineUpgradeTest(testCase) } diff --git a/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala b/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala index 529a507301a..84ab3e14d9d 100644 --- a/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/ExternalTestCaseSpec.scala @@ -5,7 +5,9 @@ import cats.data.Validated.{Invalid, Valid} import centaur.test.standard.CentaurTestCase import com.typesafe.scalalogging.StrictLogging -class ExternalTestCaseSpec(cromwellBackends: List[String]) extends AbstractCentaurTestCaseSpec(cromwellBackends) with StrictLogging { +class ExternalTestCaseSpec(cromwellBackends: List[String]) + extends AbstractCentaurTestCaseSpec(cromwellBackends) + with StrictLogging { def this() = this(CentaurTestSuite.cromwellBackends) @@ -15,11 +17,10 @@ class ExternalTestCaseSpec(cromwellBackends: List[String]) extends AbstractCenta logger.info("No external test to run") } - def runTestFile(testFile: String) = { + def runTestFile(testFile: String) = CentaurTestCase.fromFile(cromwellTracker = None)(File(testFile)) match { case Valid(testCase) => executeStandardTest(testCase) case Invalid(error) => fail(s"Invalid test case: ${error.toList.mkString(", ")}") } - } } diff --git a/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala index 300e50d937c..a63030ba40a 100644 --- a/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/PapiUpgradeTestCaseSpec.scala @@ -5,7 +5,7 @@ import org.scalatest.DoNotDiscover @DoNotDiscover class PapiUpgradeTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { + extends AbstractCromwellEngineOrBackendUpgradeTestCaseSpec(cromwellBackends) { def this() = this(CentaurTestSuite.cromwellBackends) diff --git a/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala b/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala index 456c974537a..a498807b14f 100644 --- a/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/ParallelTestCaseSpec.scala @@ -3,15 +3,15 @@ package centaur import centaur.CentaurTestSuite.cromwellTracker import org.scalatest._ - /** * Runs test cases in parallel, this should be the default type for tests unless they would otherwise crosstalk in undesirable * ways with other tests and must be made sequential. */ @DoNotDiscover class ParallelTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCentaurTestCaseSpec(cromwellBackends, cromwellTracker = cromwellTracker) with ParallelTestExecution { - + extends AbstractCentaurTestCaseSpec(cromwellBackends, cromwellTracker = cromwellTracker) + with ParallelTestExecution { + def this() = this(CentaurTestSuite.cromwellBackends) allTestCases.filter(_.testFormat.isParallel) foreach executeStandardTest diff --git a/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala b/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala index ab350f89fc1..12287d4686c 100644 --- a/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala +++ b/centaur/src/it/scala/centaur/SequentialTestCaseSpec.scala @@ -8,7 +8,9 @@ import org.scalatest.matchers.should.Matchers * such that the restarting tests execute sequentially to avoid a mayhem of Cromwell restarts */ @DoNotDiscover -class SequentialTestCaseSpec(cromwellBackends: List[String]) extends AbstractCentaurTestCaseSpec(cromwellBackends) with Matchers { +class SequentialTestCaseSpec(cromwellBackends: List[String]) + extends AbstractCentaurTestCaseSpec(cromwellBackends) + with Matchers { def this() = this(CentaurTestSuite.cromwellBackends) diff --git a/centaur/src/it/scala/centaur/WdlUpgradeTestCaseSpec.scala b/centaur/src/it/scala/centaur/WdlUpgradeTestCaseSpec.scala deleted file mode 100644 index 474bf0a7327..00000000000 --- a/centaur/src/it/scala/centaur/WdlUpgradeTestCaseSpec.scala +++ /dev/null @@ -1,13 +0,0 @@ -package centaur - -import org.scalatest.{DoNotDiscover, ParallelTestExecution} - -@DoNotDiscover -class WdlUpgradeTestCaseSpec(cromwellBackends: List[String]) - extends AbstractCentaurTestCaseSpec(cromwellBackends) with ParallelTestExecution with CentaurTestSuiteShutdown { - - def this() = this(CentaurTestSuite.cromwellBackends) - - // The WDL version upgrade tests are just regular draft-2 test cases tagged for re-use in testing the upgrade script - allTestCases.filter(CentaurTestSuite.isWdlUpgradeTest) foreach executeWdlUpgradeTest -} diff --git a/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala b/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala index b7976e64d97..6d3888767b9 100644 --- a/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala +++ b/centaur/src/it/scala/centaur/callcaching/CromwellDatabaseCallCaching.scala @@ -11,7 +11,6 @@ object CromwellDatabaseCallCaching extends StrictLogging { private val cromwellDatabase = CromwellDatabase.instance - def clearCachedResults(workflowId: String)(implicit executionContext: ExecutionContext): IO[Unit] = { + def clearCachedResults(workflowId: String)(implicit executionContext: ExecutionContext): IO[Unit] = IO.fromFuture(IO(cromwellDatabase.engineDatabase.invalidateCallCacheEntryIdsForWorkflowId(workflowId))) - } } diff --git a/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala b/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala index e9dacd18987..5b8affee887 100644 --- a/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala +++ b/centaur/src/it/scala/centaur/reporting/AggregatedIo.scala @@ -11,6 +11,7 @@ import cats.syntax.traverse._ * Validation that aggregates multiple throwable errors. */ object AggregatedIo { + /** * Similar to common.validation.ErrorOr#ErrorOr, but retains the stack traces. */ @@ -39,16 +40,16 @@ object AggregatedIo { /** * Creates an aggregated exception for multiple exceptions. */ - class AggregatedException private[reporting](exceptionContext: String, suppressed: List[Throwable]) - extends RuntimeException( - { - val suppressedZipped = suppressed.zipWithIndex - val messages = suppressedZipped map { - case (throwable, index) => s"\n ${index+1}: ${throwable.getMessage}" + class AggregatedException private[reporting] (exceptionContext: String, suppressed: List[Throwable]) + extends RuntimeException( + { + val suppressedZipped = suppressed.zipWithIndex + val messages = suppressedZipped map { case (throwable, index) => + s"\n ${index + 1}: ${throwable.getMessage}" + } + s"$exceptionContext:$messages" } - s"$exceptionContext:$messages" - } - ) { + ) { suppressed foreach addSuppressed } diff --git a/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala b/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala index 16463742632..b2335e4a794 100644 --- a/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/BigQueryReporter.scala @@ -14,7 +14,14 @@ import com.google.api.gax.retrying.RetrySettings import com.google.api.services.bigquery.BigqueryScopes import com.google.auth.Credentials import com.google.cloud.bigquery.InsertAllRequest.RowToInsert -import com.google.cloud.bigquery.{BigQuery, BigQueryError, BigQueryOptions, InsertAllRequest, InsertAllResponse, TableId} +import com.google.cloud.bigquery.{ + BigQuery, + BigQueryError, + BigQueryOptions, + InsertAllRequest, + InsertAllResponse, + TableId +} import common.util.TimeUtil._ import common.validation.Validation._ import cromwell.cloudsupport.gcp.GoogleConfiguration @@ -35,8 +42,9 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe override lazy val destination: String = bigQueryProjectOption.map(_ + "/").getOrElse("") + bigQueryDataset - private val retrySettings: RetrySettings = { - RetrySettings.newBuilder() + private val retrySettings: RetrySettings = + RetrySettings + .newBuilder() .setMaxAttempts(3) .setTotalTimeout(Duration.ofSeconds(30)) .setInitialRetryDelay(Duration.ofMillis(100)) @@ -46,7 +54,6 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe .setRpcTimeoutMultiplier(1.1) .setMaxRpcTimeout(Duration.ofSeconds(5)) .build() - } private val bigQueryCredentials: Credentials = GoogleConfiguration .apply(params.rootConfig) @@ -54,7 +61,8 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe .unsafe .credentials(Set(BigqueryScopes.BIGQUERY_INSERTDATA)) - private val bigQuery: BigQuery = BigQueryOptions.newBuilder() + private val bigQuery: BigQuery = BigQueryOptions + .newBuilder() .setRetrySettings(retrySettings) .setCredentials(bigQueryCredentials) .build() @@ -65,21 +73,19 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe private val jobKeyValueTableId = bigQueryTable("job_key_value") private val metadataTableId = bigQueryTable("metadata") - def bigQueryTable(table: String): TableId = { + def bigQueryTable(table: String): TableId = bigQueryProjectOption match { case Some(project) => TableId.of(project, bigQueryDataset, table) case None => TableId.of(bigQueryDataset, table) } - } /** * In this ErrorReporter implementation this method will send information about exceptions of type * CentaurTestException to BigQuery. Exceptions of other types will be ignored. */ - override def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + override def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = throwable match { case centaurTestException: CentaurTestException => for { @@ -98,57 +104,62 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe case _ => IO.unit // this ErrorReporter only supports exceptions of CentaurTestException type } - } private def sendBigQueryFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, centaurTestException: CentaurTestException, callAttemptFailures: Vector[CallAttemptFailure], jobKeyValueEntries: Seq[JobKeyValueEntry], - metadataEntries: Seq[MetadataEntry]): IO[Unit] = { + metadataEntries: Seq[MetadataEntry] + ): IO[Unit] = { val metadata: IO[List[BigQueryError]] = { - val metadataRows: List[util.List[RowToInsert]] = metadataEntries.map(toMetadataRow).grouped(10000).map(_.asJava).toList + val metadataRows: List[util.List[RowToInsert]] = + metadataEntries.map(toMetadataRow).grouped(10000).map(_.asJava).toList val metadataRequest: List[InsertAllRequest] = metadataRows.map(InsertAllRequest.of(metadataTableId, _)) if (metadataEntries.nonEmpty) - metadataRequest.traverse[IO, List[BigQueryError]](req => IO(bigQuery.insertAll(req)).map(_.getErrors)).map(_.flatten) + metadataRequest + .traverse[IO, List[BigQueryError]](req => IO(bigQuery.insertAll(req)).map(_.getErrors)) + .map(_.flatten) else - IO{Nil} + IO(Nil) } (IO { - val testFailureRow = toTestFailureRow(testEnvironment, ciEnvironment, centaurTestException) - val callAttemptFailureRows = callAttemptFailures.map(toCallAttemptFailureRow).asJava - val jobKeyValueRows = jobKeyValueEntries.map(toJobKeyValueRow).asJava - - val testFailureRequest = InsertAllRequest.of(testFailureTableId, testFailureRow) - val callAttemptFailuresRequest = InsertAllRequest.of(callAttemptFailureTableId, callAttemptFailureRows) - val jobKeyValuesRequest = InsertAllRequest.of(jobKeyValueTableId, jobKeyValueRows) - val testFailureErrors = bigQuery.insertAll(testFailureRequest).getErrors - val callAttemptFailuresErrors = - if (callAttemptFailures.nonEmpty) bigQuery.insertAll(callAttemptFailuresRequest).getErrors else Nil - val jobKeyValuesErrors = - if (jobKeyValueEntries.nonEmpty) bigQuery.insertAll(jobKeyValuesRequest).getErrors else Nil - - testFailureErrors ++ callAttemptFailuresErrors ++ jobKeyValuesErrors - }, metadata).mapN(_ ++ _).flatMap { + val testFailureRow = toTestFailureRow(testEnvironment, ciEnvironment, centaurTestException) + val callAttemptFailureRows = callAttemptFailures.map(toCallAttemptFailureRow).asJava + val jobKeyValueRows = jobKeyValueEntries.map(toJobKeyValueRow).asJava + + val testFailureRequest = InsertAllRequest.of(testFailureTableId, testFailureRow) + val callAttemptFailuresRequest = InsertAllRequest.of(callAttemptFailureTableId, callAttemptFailureRows) + val jobKeyValuesRequest = InsertAllRequest.of(jobKeyValueTableId, jobKeyValueRows) + val testFailureErrors = bigQuery.insertAll(testFailureRequest).getErrors + val callAttemptFailuresErrors = + if (callAttemptFailures.nonEmpty) bigQuery.insertAll(callAttemptFailuresRequest).getErrors else Nil + val jobKeyValuesErrors = + if (jobKeyValueEntries.nonEmpty) bigQuery.insertAll(jobKeyValuesRequest).getErrors else Nil + + testFailureErrors ++ callAttemptFailuresErrors ++ jobKeyValuesErrors + }, + metadata + ).mapN(_ ++ _).flatMap { case errors if errors.isEmpty => IO.unit - case errors => IO.raiseError { - val errorCount = errors.size - val threeErrors = errors.map(String.valueOf).distinct.sorted.take(3) - val continued = if (errorCount > 3) "\n..." else "" - val message = threeErrors.mkString( - s"$errorCount error(s) occurred uploading to BigQuery: \n", - "\n", - continued) - new RuntimeException(message) - } + case errors => + IO.raiseError { + val errorCount = errors.size + val threeErrors = errors.map(String.valueOf).distinct.sorted.take(3) + val continued = if (errorCount > 3) "\n..." else "" + val message = + threeErrors.mkString(s"$errorCount error(s) occurred uploading to BigQuery: \n", "\n", continued) + new RuntimeException(message) + } } } private def toTestFailureRow(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, - centaurTestException: CentaurTestException): RowToInsert = { + centaurTestException: CentaurTestException + ): RowToInsert = RowToInsert of Map( "ci_env_branch" -> ciEnvironment.branch, "ci_env_event" -> ciEnvironment.event, @@ -165,13 +176,12 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe "test_name" -> Option(testEnvironment.testCase.name), "test_stack_trace" -> Option(ExceptionUtils.getStackTrace(centaurTestException)), "test_timestamp" -> Option(OffsetDateTime.now.toUtcMilliString), - "test_workflow_id" -> centaurTestException.workflowIdOption, - ).collect { - case (key, Some(value)) => (key, value) + "test_workflow_id" -> centaurTestException.workflowIdOption + ).collect { case (key, Some(value)) => + (key, value) }.asJava - } - private def toCallAttemptFailureRow(callAttemptFailure: CallAttemptFailure): RowToInsert = { + private def toCallAttemptFailureRow(callAttemptFailure: CallAttemptFailure): RowToInsert = RowToInsert of Map( "call_fully_qualified_name" -> Option(callAttemptFailure.callFullyQualifiedName), "call_root" -> callAttemptFailure.callRootOption, @@ -182,24 +192,22 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe "start" -> callAttemptFailure.startOption.map(_.toUtcMilliString), "stderr" -> callAttemptFailure.stderrOption, "stdout" -> callAttemptFailure.stdoutOption, - "workflow_id" -> Option(callAttemptFailure.workflowId), - ).collect { - case (key, Some(value)) => (key, value) + "workflow_id" -> Option(callAttemptFailure.workflowId) + ).collect { case (key, Some(value)) => + (key, value) }.asJava - } - private def toJobKeyValueRow(jobKeyValueEntry: JobKeyValueEntry): RowToInsert = { - RowToInsert of Map( + private def toJobKeyValueRow(jobKeyValueEntry: JobKeyValueEntry): RowToInsert = + RowToInsert of Map[String, Any]( "call_fully_qualified_name" -> jobKeyValueEntry.callFullyQualifiedName, "job_attempt" -> jobKeyValueEntry.jobAttempt, "job_index" -> jobKeyValueEntry.jobIndex, "store_key" -> jobKeyValueEntry.storeKey, "store_value" -> jobKeyValueEntry.storeValue, - "workflow_execution_uuid" -> jobKeyValueEntry.workflowExecutionUuid, + "workflow_execution_uuid" -> jobKeyValueEntry.workflowExecutionUuid ).asJava - } - private def toMetadataRow(metadataEntry: MetadataEntry): RowToInsert = { + private def toMetadataRow(metadataEntry: MetadataEntry): RowToInsert = RowToInsert of Map( "call_fully_qualified_name" -> metadataEntry.callFullyQualifiedName, "job_attempt" -> metadataEntry.jobAttempt, @@ -208,11 +216,10 @@ class BigQueryReporter(override val params: ErrorReporterParams) extends ErrorRe "metadata_timestamp" -> Option(metadataEntry.metadataTimestamp.toSystemOffsetDateTime.toUtcMilliString), "metadata_value" -> metadataEntry.metadataValue.map(_.toRawString), "metadata_value_type" -> metadataEntry.metadataValueType, - "workflow_execution_uuid" -> Option(metadataEntry.workflowExecutionUuid), - ).collect { - case (key, Some(value)) => (key, value) + "workflow_execution_uuid" -> Option(metadataEntry.workflowExecutionUuid) + ).collect { case (key, Some(value)) => + (key, value) }.asJava - } } object BigQueryReporter { diff --git a/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala b/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala index 6750925ca64..0845d8b8c94 100644 --- a/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala +++ b/centaur/src/it/scala/centaur/reporting/CiEnvironment.scala @@ -5,8 +5,7 @@ import scala.util.Try /** * Scala representation of the CI environment values defined in `test.inc.sh`. */ -case class CiEnvironment -( +case class CiEnvironment( isCi: Option[Boolean], `type`: Option[String], branch: Option[String], @@ -20,7 +19,7 @@ case class CiEnvironment ) object CiEnvironment { - def apply(): CiEnvironment = { + def apply(): CiEnvironment = new CiEnvironment( isCi = sys.env.get("CROMWELL_BUILD_IS_CI").flatMap(tryToBoolean), `type` = sys.env.get("CROMWELL_BUILD_TYPE"), @@ -31,9 +30,8 @@ object CiEnvironment { provider = sys.env.get("CROMWELL_BUILD_PROVIDER"), os = sys.env.get("CROMWELL_BUILD_OS"), url = sys.env.get("CROMWELL_BUILD_URL"), - centaurType = sys.env.get("CROMWELL_BUILD_CENTAUR_TYPE"), + centaurType = sys.env.get("CROMWELL_BUILD_CENTAUR_TYPE") ) - } /** Try converting the value to a boolean, or return None. */ private def tryToBoolean(string: String): Option[Boolean] = Try(string.toBoolean).toOption diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala index 8481bf94c51..c654ad3cfc0 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporter.scala @@ -8,6 +8,7 @@ import scala.concurrent.ExecutionContext * Reports errors during testing. */ trait ErrorReporter { + /** The various parameters for this reporter. */ def params: ErrorReporterParams @@ -15,8 +16,7 @@ trait ErrorReporter { def destination: String /** Send a report of a failure. */ - def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] + def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] } diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala index 89a3e5210b8..79e13c47700 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporterCromwellDatabase.scala @@ -11,25 +11,25 @@ class ErrorReporterCromwellDatabase(cromwellDatabase: CromwellDatabase) { import centaur.TestContext._ - def jobKeyValueEntriesIo(workflowExecutionUuidOption: Option[String]) - (implicit executionContext: ExecutionContext): IO[Seq[JobKeyValueEntry]] = { + def jobKeyValueEntriesIo(workflowExecutionUuidOption: Option[String])(implicit + executionContext: ExecutionContext + ): IO[Seq[JobKeyValueEntry]] = workflowExecutionUuidOption.map(jobKeyValueEntriesIo).getOrElse(IO.pure(Seq.empty)) - } - def jobKeyValueEntriesIo(workflowExecutionUuid: String) - (implicit executionContext: ExecutionContext): IO[Seq[JobKeyValueEntry]] = { + def jobKeyValueEntriesIo(workflowExecutionUuid: String)(implicit + executionContext: ExecutionContext + ): IO[Seq[JobKeyValueEntry]] = IO.fromFuture(IO(cromwellDatabase.engineDatabase.queryJobKeyValueEntries(workflowExecutionUuid))) - } - def metadataEntriesIo(workflowExecutionUuidOption: Option[String]) - (implicit executionContext: ExecutionContext): IO[Seq[MetadataEntry]] = { + def metadataEntriesIo(workflowExecutionUuidOption: Option[String])(implicit + executionContext: ExecutionContext + ): IO[Seq[MetadataEntry]] = workflowExecutionUuidOption.map(metadataEntriesIo).getOrElse(IO.pure(Seq.empty)) - } - def metadataEntriesIo(workflowExecutionUuid: String) - (implicit executionContext: ExecutionContext): IO[Seq[MetadataEntry]] = { + def metadataEntriesIo(workflowExecutionUuid: String)(implicit + executionContext: ExecutionContext + ): IO[Seq[MetadataEntry]] = // 30 seconds is less than production (60s as of 2018-08) but hopefully high enough to work on a CI machine with contended resources IO.fromFuture(IO(cromwellDatabase.metadataDatabase.queryMetadataEntries(workflowExecutionUuid, 30.seconds))) - } } diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala index 6e716b82942..9a403e88138 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporterParams.scala @@ -5,8 +5,7 @@ import com.typesafe.config.Config /** * Collects all of the parameters to pass to a new ErrorReporter. */ -case class ErrorReporterParams -( +case class ErrorReporterParams( name: String, rootConfig: Config, reporterConfig: Config, diff --git a/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala b/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala index 8f989e3bd1b..2058264db29 100644 --- a/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala +++ b/centaur/src/it/scala/centaur/reporting/ErrorReporters.scala @@ -25,9 +25,8 @@ class ErrorReporters(rootConfig: Config) { providersConfig.entrySet.asScala.map(_.getKey.split("\\.").toList.head).toList } - private val errorReportersIo: IO[List[ErrorReporter]] = { + private val errorReportersIo: IO[List[ErrorReporter]] = AggregatedIo.aggregateExceptions("Errors while creating ErrorReporters", errorReporterNames.map(getErrorReporter)) - } val errorReporters: List[ErrorReporter] = errorReportersIo.unsafeRunSync() @@ -42,26 +41,27 @@ class ErrorReporters(rootConfig: Config) { * @param throwable The exception that occurred while running the test. * @return An IO effect that will log the failure. */ - def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = if (errorReporters.isEmpty) { // If the there are no reporters, then just "throw" the exception. Do not retry to run the test. IO.raiseError(throwable) } else { val listIo = errorReporters.map(_.logFailure(testEnvironment, ciEnvironment, throwable)) - AggregatedIo.aggregateExceptions("Errors while reporting a failure", listIo).handleErrorWith(err => { - err.addSuppressed(throwable) - IO.raiseError(err) - }).void + AggregatedIo + .aggregateExceptions("Errors while reporting a failure", listIo) + .handleErrorWith { err => + err.addSuppressed(throwable) + IO.raiseError(err) + } + .void } - } /** * Constructs the IO reporter by name. */ - private def getErrorReporter(errorReporterName: String): IO[ErrorReporter] = { + private def getErrorReporter(errorReporterName: String): IO[ErrorReporter] = IO { val clazz = errorReporterConfig.getString(s"providers.$errorReporterName.class") val reporterConfig = errorReporterConfig.getOrElse(s"providers.$errorReporterName.config", ConfigFactory.empty) @@ -69,7 +69,6 @@ class ErrorReporters(rootConfig: Config) { val params = ErrorReporterParams(errorReporterName, rootConfig, reporterConfig, errorReporterCromwellDatabase) constructor.newInstance(params).asInstanceOf[ErrorReporter] } - } } object ErrorReporters extends StrictLogging { @@ -84,9 +83,8 @@ object ErrorReporters extends StrictLogging { if (retryAttempts > 0) logger.info("Error retry count: {}", retryAttempts) - def logFailure(testEnvironment: TestEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + def logFailure(testEnvironment: TestEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = errorReporters.logFailure(testEnvironment, ciEnvironment, throwable) - } } diff --git a/centaur/src/it/scala/centaur/reporting/GcsReporter.scala b/centaur/src/it/scala/centaur/reporting/GcsReporter.scala index 41e15d80b3b..55b5be270f4 100644 --- a/centaur/src/it/scala/centaur/reporting/GcsReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/GcsReporter.scala @@ -9,7 +9,10 @@ import net.ceedubs.ficus.Ficus._ import scala.concurrent.ExecutionContext -class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporter with SuccessReporter with StrictLogging { +class GcsReporter(override val params: ErrorReporterParams) + extends ErrorReporter + with SuccessReporter + with StrictLogging { val storage = StorageOptions.getDefaultInstance.getService val reportBucket = params.reporterConfig.as[String]("report-bucket") val reportPath = params.reporterConfig.as[String]("report-path") @@ -21,10 +24,9 @@ class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporte * In this ErrorReporter implementation this method will save information about exceptions of type * CentaurTestException to GCS. Exceptions of other types will be ignored. */ - override def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + override def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = throwable match { case centaurTestException: CentaurTestException => logger.info(s"Reporting failed metadata to gs://$reportBucket/$reportPath") @@ -32,7 +34,6 @@ class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporte case _ => IO.unit // this ErrorReporter only supports exceptions of CentaurTestException type } - } override def logSuccessfulRun(submitResponse: SubmitWorkflowResponse): IO[Unit] = { logger.info(s"Reporting successful metadata to gs://$reportBucket/$reportPath") @@ -44,7 +45,8 @@ class GcsReporter(override val params: ErrorReporterParams) extends ErrorReporte private def pushJsonToGcs(json: String) = IO { storage.create( - BlobInfo.newBuilder(reportBucket, reportPath) + BlobInfo + .newBuilder(reportBucket, reportPath) .setContentType("application/json") .build(), json.toArray.map(_.toByte) diff --git a/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala b/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala index 4332e01db24..ff5aeb5f7d5 100644 --- a/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/Slf4jReporter.scala @@ -13,15 +13,13 @@ import scala.concurrent.ExecutionContext * Useful as a backup in cases where another reporter is not available, for example in external PRs where secure * environment variables are not available. */ -class Slf4jReporter(override val params: ErrorReporterParams) - extends ErrorReporter with StrictLogging { +class Slf4jReporter(override val params: ErrorReporterParams) extends ErrorReporter with StrictLogging { override lazy val destination: String = "error" - override def logFailure(testEnvironment: TestEnvironment, - ciEnvironment: CiEnvironment, - throwable: Throwable) - (implicit executionContext: ExecutionContext): IO[Unit] = { + override def logFailure(testEnvironment: TestEnvironment, ciEnvironment: CiEnvironment, throwable: Throwable)(implicit + executionContext: ExecutionContext + ): IO[Unit] = IO { val errorMessage = throwable match { @@ -41,9 +39,9 @@ class Slf4jReporter(override val params: ErrorReporterParams) if (testEnvironment.attempt >= testEnvironment.retries) { logger.error(message, throwable) } else { - val messageWithShortExceptionContext = message + " (" + ExceptionUtils.getMessage(throwable).replace("\n", " ").take(150) + "[...])" + val messageWithShortExceptionContext = + message + " (" + ExceptionUtils.getMessage(throwable).replace("\n", " ").take(150) + "[...])" logger.warn(messageWithShortExceptionContext) } } - } } diff --git a/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala b/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala index 2453ca71898..fb0c6a389e9 100644 --- a/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala +++ b/centaur/src/it/scala/centaur/reporting/SuccessReporter.scala @@ -12,13 +12,13 @@ object SuccessReporters { * This is gross and piggy backs on the error reporting code but achieves what we need for now without a refactoring * of the error reporting code to handle both success and error reporting */ - private val successReporters: List[ErrorReporter with SuccessReporter] = ErrorReporters.errorReporters.errorReporters.collect({case s: SuccessReporter => s }) + private val successReporters: List[ErrorReporter with SuccessReporter] = + ErrorReporters.errorReporters.errorReporters.collect { case s: SuccessReporter => s } - def logSuccessfulRun(submitResponse: SubmitWorkflowResponse): IO[SubmitResponse] = { - if (successReporters.isEmpty) IO.pure(submitResponse) + def logSuccessfulRun(submitResponse: SubmitWorkflowResponse): IO[SubmitResponse] = + if (successReporters.isEmpty) IO.pure(submitResponse) else { val listIo = successReporters.map(_.logSuccessfulRun(submitResponse)) AggregatedIo.aggregateExceptions("Errors while reporting centaur success", listIo).map(_ => submitResponse) } - } } diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test new file mode 100644 index 00000000000..b7270ccfb11 --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read.test @@ -0,0 +1,24 @@ +name: azure_blob_storage_read +testFormat: workflowsuccess +backends: [Local] +tags: ["blob", "azure"] +retryTestFailures: false + +files { + workflow: azure_blob_storage_read/azure_blob_storage_read.wdl + inputs: azure_blob_storage_read/azure_blob_storage_read.inputs + options: azure_blob_storage_read/azure_blob_storage_read.options +} + +metadata { + status: Succeeded + "outputs.azure_blob_storage_read.s1": "This is my test file! Did it work??" +} + +# az:// is the root of the container specified in reference.conf. +# Here, we verify that exactly one log was written. + +fileSystemCheck: "blob" +outputExpectations: { + "az://test-cromwell-workflow-logs/workflow.<>.log" : 1 +} diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs new file mode 100644 index 00000000000..c81e166493f --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.inputs @@ -0,0 +1,3 @@ +{ + "azure_blob_storage_read.file1": "https://centaurtesting.blob.core.windows.net/test-blob/testRead.txt" +} diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options new file mode 100644 index 00000000000..8d68fcdd6bf --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.options @@ -0,0 +1,3 @@ +{ + "final_workflow_log_dir": "https://centaurtesting.blob.core.windows.net/test-blob/test-cromwell-workflow-logs" +} diff --git a/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl new file mode 100644 index 00000000000..d19d417cdd5 --- /dev/null +++ b/centaur/src/main/resources/azureBlobTestCases/azure_blob_storage_read/azure_blob_storage_read.wdl @@ -0,0 +1,12 @@ +version 1.0 + +workflow azure_blob_storage_read { + + input { + File file1 + } + + output { + String s1 = read_string(file1) + } +} diff --git a/centaur/src/main/resources/integrationTestCases/bcbio_joint_gvcf.test b/centaur/src/main/resources/integrationTestCases/bcbio_joint_gvcf.test deleted file mode 100644 index ef32cbc7d7c..00000000000 --- a/centaur/src/main/resources/integrationTestCases/bcbio_joint_gvcf.test +++ /dev/null @@ -1,23 +0,0 @@ -name: bcbio_joint_gvcf -testFormat: workflowsuccess -backends: [Papiv2] -workflowType: CWL -workflowTypeVersion: v1.0 -tags: [bcbio] - - -files { - workflow: cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint.cwl - inputs: cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint-samples.json - imports: [ - cwl/bcbio/gvcf-joint-workflow/steps, - cwl/bcbio/gvcf-joint-workflow/wf-alignment.cwl, - cwl/bcbio/gvcf-joint-workflow/wf-jointcall.cwl, - cwl/bcbio/gvcf-joint-workflow/wf-variantcall.cwl - ] - options: cwl/bcbio/bcbio.options -} - -metadata { - status: Succeeded -} diff --git a/centaur/src/main/resources/integrationTestCases/bcbio_prealign.test b/centaur/src/main/resources/integrationTestCases/bcbio_prealign.test deleted file mode 100644 index 2c4a6db3f45..00000000000 --- a/centaur/src/main/resources/integrationTestCases/bcbio_prealign.test +++ /dev/null @@ -1,20 +0,0 @@ -name: bcbio_prealign -testFormat: workflowsuccess -backends: [Papiv2] -workflowType: CWL -workflowTypeVersion: v1.0 -tags: [bcbio] - -files { - workflow: cwl/bcbio/prealign-workflow/main-prealign.cwl - inputs: cwl/bcbio/prealign-workflow/main-prealign-samples.json - imports: [ - cwl/bcbio/prealign-workflow/steps, - cwl/bcbio/prealign-workflow/wf-variantcall.cwl - ] - options: cwl/bcbio/bcbio.options -} - -metadata { - status: Succeeded -} diff --git a/centaur/src/main/resources/integrationTestCases/bcbio_rnaseq.test b/centaur/src/main/resources/integrationTestCases/bcbio_rnaseq.test deleted file mode 100644 index 64ac876c568..00000000000 --- a/centaur/src/main/resources/integrationTestCases/bcbio_rnaseq.test +++ /dev/null @@ -1,19 +0,0 @@ -name: bcbio_rnaseq -testFormat: workflowsuccess -backends: [Papiv2] -workflowType: CWL -workflowTypeVersion: v1.0 -tags: [bcbio] - -files { - workflow: cwl/bcbio/rnaseq-workflow/main-rnaseq.cwl - inputs: cwl/bcbio/rnaseq-workflow/main-rnaseq-samples.json - imports: [ - cwl/bcbio/rnaseq-workflow/steps - ] - options: cwl/bcbio/bcbio.options -} - -metadata { - status: Succeeded -} diff --git a/centaur/src/main/resources/integrationTestCases/bcbio_somatic.test b/centaur/src/main/resources/integrationTestCases/bcbio_somatic.test deleted file mode 100644 index 440099f280e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/bcbio_somatic.test +++ /dev/null @@ -1,22 +0,0 @@ -name: bcbio_somatic -testFormat: workflowsuccess -backends: [Papiv2] -workflowType: CWL -workflowTypeVersion: v1.0 -tags: [bcbio] - -files { - workflow: cwl/bcbio/somatic-workflow/main-somatic.cwl - inputs: cwl/bcbio/somatic-workflow/main-somatic-samples.json - imports: [ - cwl/bcbio/somatic-workflow/steps, - cwl/bcbio/somatic-workflow/wf-alignment.cwl, - cwl/bcbio/somatic-workflow/wf-svcall.cwl, - cwl/bcbio/somatic-workflow/wf-variantcall.cwl - ] - options: cwl/bcbio/bcbio.options -} - -metadata { - status: Succeeded -} diff --git a/centaur/src/main/resources/integrationTestCases/bcbio_svcall.test b/centaur/src/main/resources/integrationTestCases/bcbio_svcall.test deleted file mode 100644 index 9116814fe6d..00000000000 --- a/centaur/src/main/resources/integrationTestCases/bcbio_svcall.test +++ /dev/null @@ -1,22 +0,0 @@ -name: bcbio_svcall -testFormat: workflowsuccess -backends: [Papiv2] -workflowType: CWL -workflowTypeVersion: v1.0 -tags: [bcbio] - -files { - workflow: cwl/bcbio/svcall-workflow/main-svcall.cwl - inputs: cwl/bcbio/svcall-workflow/main-svcall-samples.json - imports: [ - cwl/bcbio/svcall-workflow/steps, - cwl/bcbio/svcall-workflow/wf-alignment.cwl, - cwl/bcbio/svcall-workflow/wf-svcall.cwl, - cwl/bcbio/svcall-workflow/wf-variantcall.cwl - ] - options: cwl/bcbio/bcbio.options -} - -metadata { - status: Succeeded -} diff --git a/centaur/src/main/resources/integrationTestCases/bcbio_wes-chr21-test-workflow.test b/centaur/src/main/resources/integrationTestCases/bcbio_wes-chr21-test-workflow.test deleted file mode 100644 index 1969d6bef66..00000000000 --- a/centaur/src/main/resources/integrationTestCases/bcbio_wes-chr21-test-workflow.test +++ /dev/null @@ -1,22 +0,0 @@ -name: wes_chr21_test-workflow -testFormat: workflowsuccess -backends: [Papiv2] -workflowType: CWL -workflowTypeVersion: v1.0 -tags: [bcbio] - -files { - workflow: cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test.cwl - inputs: cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test-samples.json - imports: [ - cwl/bcbio/wes_chr21_test-workflow-gcp/steps, - cwl/bcbio/wes_chr21_test-workflow-gcp/wf-alignment.cwl, - cwl/bcbio/wes_chr21_test-workflow-gcp/wf-svcall.cwl, - cwl/bcbio/wes_chr21_test-workflow-gcp/wf-variantcall.cwl - ] - options: cwl/bcbio/bcbio.options -} - -metadata { - status: Succeeded -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/bcbio.options b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/bcbio.options deleted file mode 100644 index 1340bda7f1c..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/bcbio.options +++ /dev/null @@ -1,5 +0,0 @@ -{ - "default_runtime_attributes": { - "bootDiskSizeGb": 20 - } -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint-samples.json b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint-samples.json deleted file mode 100644 index b7080e81448..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint-samples.json +++ /dev/null @@ -1,615 +0,0 @@ -{ - "analysis": [ - "variant2", - "variant2" - ], - "config__algorithm__adapters": [ - [ - "polyx" - ], - [ - "polyx" - ] - ], - "config__algorithm__align_split_size": [ - null, - null - ], - "config__algorithm__aligner": [ - "bwa", - "bwa" - ], - "config__algorithm__bam_clean": [ - "False", - "False" - ], - "config__algorithm__coverage": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/coverage_transcripts-bam.bed" - }, - null - ], - "config__algorithm__coverage_interval": [ - null, - null - ], - "config__algorithm__effects": [ - "snpeff", - "snpeff" - ], - "config__algorithm__ensemble": [ - null, - null - ], - "config__algorithm__exclude_regions": [ - [], - [] - ], - "config__algorithm__mark_duplicates": [ - "True", - "True" - ], - "config__algorithm__min_allele_fraction": [ - 10.0, - 10.0 - ], - "config__algorithm__nomap_split_size": [ - 250, - 250 - ], - "config__algorithm__nomap_split_targets": [ - 20, - 20 - ], - "config__algorithm__qc": [ - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "samtools", - "variants" - ], - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "samtools", - "variants" - ] - ], - "config__algorithm__recalibrate": [ - "True", - "True" - ], - "config__algorithm__tools_off": [ - [], - [] - ], - "config__algorithm__tools_on": [ - [ - "gvcf" - ], - [ - "gvcf" - ] - ], - "config__algorithm__trim_reads": [ - "read_through", - "False" - ], - "config__algorithm__validate": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/reference_material/7_100326_FC6107FAAXX-grade.vcf" - }, - null - ], - "config__algorithm__validate_regions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - }, - null - ], - "config__algorithm__variant_regions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - } - ], - "config__algorithm__variantcaller": [ - [ - "gatk-haplotype", - "strelka2" - ], - [ - "gatk-haplotype", - "strelka2" - ] - ], - "config__algorithm__vcfanno": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.conf" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.lua" - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.conf" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.lua" - } - ] - ], - "description": [ - "Test1", - "Test2" - ], - "files": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam.bai" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/6_100326_FC6107FAAXX.bam" - } - ] - ], - "genome_build": [ - "hg19", - "hg19" - ], - "genome_resources__aliases__ensembl": [ - "homo_sapiens_vep_83_GRCh37", - "homo_sapiens_vep_83_GRCh37" - ], - "genome_resources__aliases__human": [ - "True", - "True" - ], - "genome_resources__aliases__snpeff": [ - "hg19", - "hg19" - ], - "genome_resources__rnaseq__gene_bed": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - } - ], - "genome_resources__variation__1000g": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__clinvar": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__cosmic": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__dbsnp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__encode_blacklist": [ - null, - null - ], - "genome_resources__variation__esp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__exac": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__gnomad_exome": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__lcr": [ - null, - null - ], - "genome_resources__variation__polyx": [ - null, - null - ], - "genome_resources__variation__train_hapmap": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__train_indels": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - } - ], - "metadata__batch": [ - "b1", - "b1" - ], - "metadata__phenotype": [ - "", - "" - ], - "reference__bwa__indexes": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.sa" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.sa" - } - ] - } - ], - "reference__fasta__base": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - } - ], - "reference__genome_context": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ] - ], - "reference__rtg": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - } - ], - "reference__snpeff__hg19": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - } - ], - "reference__versions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - } - ], - "resources": [ - "{}", - "{}" - ], - "rgnames__lane": [ - "Test1", - "Test2" - ], - "rgnames__lb": [ - null, - null - ], - "rgnames__pl": [ - "illumina", - "illumina" - ], - "rgnames__pu": [ - "Test1", - "Test2" - ], - "rgnames__rg": [ - "Test1", - "Test2" - ], - "rgnames__sample": [ - "Test1", - "Test2" - ], - "vrn_file": [ - null, - null - ] -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint.cwl deleted file mode 100644 index a798100dc12..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/main-gvcf-joint.cwl +++ /dev/null @@ -1,794 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: files - type: - items: - items: File - type: array - type: array -- id: config__algorithm__trim_reads - type: - items: - - string - - 'null' - - boolean - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__vcfanno - type: - items: - items: File - type: array - type: array -- id: resources - type: - items: string - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__adapters - type: - items: - items: string - type: array - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__versions - type: - items: File - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__validate - type: - items: - - File - - 'null' - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: description - type: - items: string - type: array -- id: config__algorithm__validate_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__rtg - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -outputs: -- id: rgnames__sample_out - outputSource: prep_samples/rgnames__sample - type: - items: string - type: array -- id: align_bam - outputSource: postprocess_alignment/align_bam - type: - items: - - File - - 'null' - type: array -- id: regions__sample_callable - outputSource: postprocess_alignment/regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: validate__grading_summary - outputSource: summarize_vc/validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: variants__calls - outputSource: summarize_vc/variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - outputSource: summarize_vc/variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: summary__multiqc - outputSource: multiqc_summary/summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - outputSource: multiqc_summary/versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - outputSource: multiqc_summary/versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: alignment_to_rec - in: - - id: files - source: files - - id: analysis - source: analysis - - id: config__algorithm__align_split_size - source: config__algorithm__align_split_size - - id: reference__fasta__base - source: reference__fasta__base - - id: rgnames__pl - source: rgnames__pl - - id: rgnames__sample - source: rgnames__sample - - id: rgnames__pu - source: rgnames__pu - - id: rgnames__lane - source: rgnames__lane - - id: rgnames__rg - source: rgnames__rg - - id: rgnames__lb - source: rgnames__lb - - id: reference__bwa__indexes - source: reference__bwa__indexes - - id: config__algorithm__aligner - source: config__algorithm__aligner - - id: config__algorithm__trim_reads - source: config__algorithm__trim_reads - - id: config__algorithm__adapters - source: config__algorithm__adapters - - id: config__algorithm__bam_clean - source: config__algorithm__bam_clean - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: config__algorithm__mark_duplicates - source: config__algorithm__mark_duplicates - - id: resources - source: resources - - id: description - source: description - out: - - id: alignment_rec - run: steps/alignment_to_rec.cwl -- id: alignment - in: - - id: alignment_rec - source: alignment_to_rec/alignment_rec - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: wf-alignment.cwl - scatter: - - alignment_rec - scatterMethod: dotproduct -- id: prep_samples_to_rec - in: - - id: config__algorithm__coverage - source: config__algorithm__coverage - - id: rgnames__sample - source: rgnames__sample - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: prep_samples_rec - run: steps/prep_samples_to_rec.cwl -- id: prep_samples - in: - - id: prep_samples_rec - source: prep_samples_to_rec/prep_samples_rec - out: - - id: rgnames__sample - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - run: steps/prep_samples.cwl - scatter: - - prep_samples_rec - scatterMethod: dotproduct -- id: postprocess_alignment_to_rec - in: - - id: align_bam - source: alignment/align_bam - - id: config__algorithm__coverage_interval - source: config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: prep_samples/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: prep_samples/config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - source: prep_samples/config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - source: prep_samples/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: prep_samples/config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - source: prep_samples/config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - source: prep_samples/config__algorithm__seq2c_bed_ready - - id: config__algorithm__recalibrate - source: config__algorithm__recalibrate - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: postprocess_alignment_rec - run: steps/postprocess_alignment_to_rec.cwl -- id: postprocess_alignment - in: - - id: postprocess_alignment_rec - source: postprocess_alignment_to_rec/postprocess_alignment_rec - out: - - id: config__algorithm__coverage_interval - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - - id: regions__callable - - id: regions__sample_callable - - id: regions__nblock - - id: depth__samtools__stats - - id: depth__samtools__idxstats - - id: depth__variant_regions__regions - - id: depth__variant_regions__dist - - id: depth__sv_regions__regions - - id: depth__sv_regions__dist - - id: depth__coverage__regions - - id: depth__coverage__dist - - id: depth__coverage__thresholds - - id: align_bam - run: steps/postprocess_alignment.cwl - scatter: - - postprocess_alignment_rec - scatterMethod: dotproduct -- id: combine_sample_regions - in: - - id: regions__callable - source: postprocess_alignment/regions__callable - - id: regions__nblock - source: postprocess_alignment/regions__nblock - - id: metadata__batch - source: metadata__batch - - id: config__algorithm__nomap_split_size - source: config__algorithm__nomap_split_size - - id: config__algorithm__nomap_split_targets - source: config__algorithm__nomap_split_targets - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: config__algorithm__callable_regions - - id: config__algorithm__non_callable_regions - - id: config__algorithm__callable_count - run: steps/combine_sample_regions.cwl -- id: batch_for_variantcall - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: align_bam - source: postprocess_alignment/align_bam - - id: vrn_file - source: vrn_file - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: config__algorithm__variantcaller - source: config__algorithm__variantcaller - - id: config__algorithm__ensemble - source: config__algorithm__ensemble - - id: config__algorithm__vcfanno - source: config__algorithm__vcfanno - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__effects - source: config__algorithm__effects - - id: config__algorithm__min_allele_fraction - source: config__algorithm__min_allele_fraction - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__validate - source: config__algorithm__validate - - id: config__algorithm__validate_regions - source: config__algorithm__validate_regions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__rtg - source: reference__rtg - - id: reference__genome_context - source: reference__genome_context - - id: genome_resources__variation__clinvar - source: genome_resources__variation__clinvar - - id: genome_resources__variation__cosmic - source: genome_resources__variation__cosmic - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__esp - source: genome_resources__variation__esp - - id: genome_resources__variation__exac - source: genome_resources__variation__exac - - id: genome_resources__variation__gnomad_exome - source: genome_resources__variation__gnomad_exome - - id: genome_resources__variation__1000g - source: genome_resources__variation__1000g - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__aliases__ensembl - source: genome_resources__aliases__ensembl - - id: genome_resources__aliases__human - source: genome_resources__aliases__human - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__hg19 - source: reference__snpeff__hg19 - - id: genome_resources__variation__train_hapmap - source: genome_resources__variation__train_hapmap - - id: genome_resources__variation__train_indels - source: genome_resources__variation__train_indels - - id: resources - source: resources - - id: description - source: description - out: - - id: batch_rec - run: steps/batch_for_variantcall.cwl -- id: variantcall - in: - - id: batch_rec - source: batch_for_variantcall/batch_rec - out: - - id: vc_rec - run: wf-variantcall.cwl - scatter: - - batch_rec - scatterMethod: dotproduct -- id: batch_for_jointvc - in: - - id: vc_rec - source: variantcall/vc_rec - out: - - id: jointvc_batch_rec - run: steps/batch_for_jointvc.cwl -- id: jointcall - in: - - id: jointvc_batch_rec - source: batch_for_jointvc/jointvc_batch_rec - out: - - id: jointvc_rec - - id: vrn_file_joint - run: wf-jointcall.cwl - scatter: - - jointvc_batch_rec - scatterMethod: dotproduct -- id: summarize_vc - in: - - id: jointvc_rec - source: jointcall/jointvc_rec - out: - - id: variants__calls - - id: variants__gvcf - - id: variants__samples - - id: validate__grading_summary - - id: validate__grading_plots - run: steps/summarize_vc.cwl -- id: qc_to_rec - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: analysis - source: analysis - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__versions - source: reference__versions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: genome_build - source: genome_build - - id: config__algorithm__qc - source: config__algorithm__qc - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: depth__variant_regions__dist - source: postprocess_alignment/depth__variant_regions__dist - - id: depth__samtools__stats - source: postprocess_alignment/depth__samtools__stats - - id: depth__samtools__idxstats - source: postprocess_alignment/depth__samtools__idxstats - - id: depth__sv_regions__regions - source: postprocess_alignment/depth__sv_regions__regions - - id: depth__sv_regions__dist - source: postprocess_alignment/depth__sv_regions__dist - - id: depth__coverage__regions - source: postprocess_alignment/depth__coverage__regions - - id: depth__coverage__dist - source: postprocess_alignment/depth__coverage__dist - - id: depth__coverage__thresholds - source: postprocess_alignment/depth__coverage__thresholds - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__coverage - source: postprocess_alignment/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: postprocess_alignment/config__algorithm__coverage_merged - - id: variants__samples - source: summarize_vc/variants__samples - - id: resources - source: resources - - id: description - source: description - out: - - id: qc_rec - run: steps/qc_to_rec.cwl -- id: pipeline_summary - in: - - id: qc_rec - source: qc_to_rec/qc_rec - out: - - id: qcout_rec - run: steps/pipeline_summary.cwl - scatter: - - qc_rec - scatterMethod: dotproduct -- id: multiqc_summary - in: - - id: qcout_rec - source: pipeline_summary/qcout_rec - out: - - id: summary__multiqc - - id: versions__tools - - id: versions__data - run: steps/multiqc_summary.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/alignment_to_rec.cwl deleted file mode 100644 index 1379c575535..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/alignment_to_rec.cwl +++ /dev/null @@ -1,198 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=alignment_rec:resources;description;config__algorithm__align_split_size;files;config__algorithm__trim_reads;reference__fasta__base;config__algorithm__adapters;rgnames__lb;rgnames__rg;rgnames__lane;reference__bwa__indexes;config__algorithm__bam_clean;config__algorithm__aligner;rgnames__pl;rgnames__pu;config__algorithm__mark_duplicates;analysis;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=files:var,analysis:var,config__algorithm__align_split_size:var,reference__fasta__base:var,rgnames__pl:var,rgnames__sample:var,rgnames__pu:var,rgnames__lane:var,rgnames__rg:var,rgnames__lb:var,reference__bwa__indexes:var,config__algorithm__aligner:var,config__algorithm__trim_reads:var,config__algorithm__adapters:var,config__algorithm__bam_clean:var,config__algorithm__variant_regions:var,config__algorithm__mark_duplicates:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: files - type: - items: - items: File - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: config__algorithm__trim_reads - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__adapters - type: - items: - items: string - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/batch_for_jointvc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/batch_for_jointvc.cwl deleted file mode 100644 index 523567ed191..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/batch_for_jointvc.cwl +++ /dev/null @@ -1,329 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=jointvc_batch_rec:resources;description;batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;vrn_file;reference__fasta__base;metadata__phenotype;config__algorithm__vcfanno;config__algorithm__variantcaller;genome_resources__variation__1000g;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;config__algorithm__min_allele_fraction;genome_resources__variation__train_hapmap;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=vc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_jointvc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: vc_rec - type: - items: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array - type: array -outputs: -- id: jointvc_batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: jointvc_batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/batch_for_variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/batch_for_variantcall.cwl deleted file mode 100644 index ca40e5b30cf..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/batch_for_variantcall.cwl +++ /dev/null @@ -1,401 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=batch_rec:resources;description;reference__fasta__base;metadata__phenotype;config__algorithm__vcfanno;config__algorithm__variantcaller;genome_resources__variation__1000g;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;config__algorithm__min_allele_fraction;vrn_file;genome_resources__variation__train_hapmap;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;align_bam;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,align_bam:var,vrn_file:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,regions__sample_callable:var,config__algorithm__variantcaller:var,config__algorithm__ensemble:var,config__algorithm__vcfanno:var,config__algorithm__coverage_interval:var,config__algorithm__effects:var,config__algorithm__min_allele_fraction:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__validate:var,config__algorithm__validate_regions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,reference__fasta__base:var,reference__rtg:var,reference__genome_context:var,genome_resources__variation__clinvar:var,genome_resources__variation__cosmic:var,genome_resources__variation__dbsnp:var,genome_resources__variation__esp:var,genome_resources__variation__exac:var,genome_resources__variation__gnomad_exome:var,genome_resources__variation__1000g:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__aliases__ensembl:var,genome_resources__aliases__human:var,genome_resources__aliases__snpeff:var,reference__snpeff__hg19:var,genome_resources__variation__train_hapmap:var,genome_resources__variation__train_indels:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_variantcall -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 2048 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__vcfanno - type: - items: - items: File - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__validate - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__validate_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__rtg - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/combine_sample_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/combine_sample_regions.cwl deleted file mode 100644 index d876db22bd2..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/combine_sample_regions.cwl +++ /dev/null @@ -1,99 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=config__algorithm__callable_regions,config__algorithm__non_callable_regions,config__algorithm__callable_count -- sentinel_inputs=regions__callable:var,regions__nblock:var,metadata__batch:var,config__algorithm__nomap_split_size:var,config__algorithm__nomap_split_targets:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- combine_sample_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 2048 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: regions__callable - type: - items: - - File - - 'null' - type: array -- id: regions__nblock - type: - items: - - File - - 'null' - type: array -- id: metadata__batch - type: - items: string - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__non_callable_regions - type: - items: File - type: array -- id: config__algorithm__callable_count - type: - items: int - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/compare_to_rm.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/compare_to_rm.cwl deleted file mode 100644 index 06152084c58..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/compare_to_rm.cwl +++ /dev/null @@ -1,337 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vc_rec:batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;resources;description;vrn_file;reference__fasta__base;metadata__phenotype;config__algorithm__vcfanno;config__algorithm__variantcaller;genome_resources__variation__1000g;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;config__algorithm__min_allele_fraction;genome_resources__variation__train_hapmap;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- compare_to_rm -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 4096 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: gvcf-regions - specs: - - https://anaconda.org/bioconda/gvcf-regions - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: rtg-tools - specs: - - https://anaconda.org/bioconda/rtg-tools - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file - secondaryFiles: - - .tbi - type: File -outputs: -- id: vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/concat_batch_variantcalls.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/concat_batch_variantcalls.cwl deleted file mode 100644 index d486086c79e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/concat_batch_variantcalls.cwl +++ /dev/null @@ -1,196 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-merge -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,region_block:var,vrn_file_region:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- concat_batch_variantcalls -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block - type: - items: - items: string - type: array - type: array -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/concat_batch_variantcalls_jointvc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/concat_batch_variantcalls_jointvc.cwl deleted file mode 100644 index dbb0f54be94..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/concat_batch_variantcalls_jointvc.cwl +++ /dev/null @@ -1,209 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-merge -- sentinel_outputs=vrn_file_joint -- sentinel_inputs=jointvc_batch_rec:record,region:var,vrn_file_region:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- concat_batch_variantcalls_jointvc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: jointvc_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: jointvc_batch_rec - type: record - type: array -- id: region - type: - items: string - type: array -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -outputs: -- id: vrn_file_joint - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/finalize_jointvc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/finalize_jointvc.cwl deleted file mode 100644 index fd1e04762e4..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/finalize_jointvc.cwl +++ /dev/null @@ -1,331 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=jointvc_rec:resources;description;batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;vrn_file;reference__fasta__base;metadata__phenotype;config__algorithm__vcfanno;config__algorithm__variantcaller;genome_resources__variation__1000g;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;config__algorithm__min_allele_fraction;genome_resources__variation__train_hapmap;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions;vrn_file_joint -- sentinel_inputs=jointvc_batch_rec:record,vrn_file_joint:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- finalize_jointvc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: jointvc_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: jointvc_batch_rec - type: record - type: array -- id: vrn_file_joint - secondaryFiles: - - .tbi - type: File -outputs: -- id: jointvc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - - name: vrn_file_joint - type: File - name: jointvc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/get_parallel_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/get_parallel_regions.cwl deleted file mode 100644 index 18e635c4006..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/get_parallel_regions.cwl +++ /dev/null @@ -1,171 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-split -- sentinel_outputs=region_block -- sentinel_inputs=batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- get_parallel_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 2048 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: region_block - type: - items: - items: string - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/get_parallel_regions_jointvc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/get_parallel_regions_jointvc.cwl deleted file mode 100644 index fac484f75ca..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/get_parallel_regions_jointvc.cwl +++ /dev/null @@ -1,184 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-split -- sentinel_outputs=region -- sentinel_inputs=jointvc_batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- get_parallel_regions_jointvc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: jointvc_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: jointvc_batch_rec - type: record - type: array -outputs: -- id: region - type: - items: string - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/merge_split_alignments.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/merge_split_alignments.cwl deleted file mode 100644 index 66e17dc7aa9..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/merge_split_alignments.cwl +++ /dev/null @@ -1,168 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-merge -- sentinel_outputs=align_bam,work_bam_plus__disc,work_bam_plus__sr,hla__fastq -- sentinel_inputs=alignment_rec:record,work_bam:var,align_bam:var,work_bam_plus__disc:var,work_bam_plus__sr:var,hla__fastq:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- merge_split_alignments -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1035 - ramMin: 4096 - tmpdirMin: 6 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -- id: work_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: align_bam_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__disc_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__sr_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: hla__fastq_toolinput - type: - items: - - 'null' - - items: File - type: array - type: array -outputs: -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/multiqc_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/multiqc_summary.cwl deleted file mode 100644 index 45a90274381..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/multiqc_summary.cwl +++ /dev/null @@ -1,95 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=summary__multiqc,versions__tools,versions__data -- sentinel_inputs=qcout_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- multiqc_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 2048 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: multiqc - specs: - - https://anaconda.org/bioconda/multiqc - - package: multiqc-bcbio - specs: - - https://anaconda.org/bioconda/multiqc-bcbio -inputs: -- id: qcout_rec - type: - items: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record - type: array -outputs: -- id: summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/pipeline_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/pipeline_summary.cwl deleted file mode 100644 index 75e0c7a35d5..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/pipeline_summary.cwl +++ /dev/null @@ -1,219 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=qcout_rec:summary__qc;summary__metrics;description;reference__versions;genome_build;config__algorithm__tools_off;config__algorithm__qc;config__algorithm__tools_on -- sentinel_inputs=qc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- pipeline_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 4096 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: fastqc=0.11.7=5 - specs: - - https://anaconda.org/bioconda/fastqc=0.11.7=5 - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: hts-nim-tools - specs: - - https://anaconda.org/bioconda/hts-nim-tools - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: qsignature - specs: - - https://anaconda.org/bioconda/qsignature - - package: qualimap - specs: - - https://anaconda.org/bioconda/qualimap - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: preseq - specs: - - https://anaconda.org/bioconda/preseq - - package: peddy - specs: - - https://anaconda.org/bioconda/peddy - - package: verifybamid2 - specs: - - https://anaconda.org/bioconda/verifybamid2 -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: qc_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record -outputs: -- id: qcout_rec - type: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_alignment.cwl deleted file mode 100644 index 09728af2814..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_alignment.cwl +++ /dev/null @@ -1,223 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=config__algorithm__coverage_interval,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready,regions__callable,regions__sample_callable,regions__nblock,depth__samtools__stats,depth__samtools__idxstats,depth__variant_regions__regions,depth__variant_regions__dist,depth__sv_regions__regions,depth__sv_regions__dist,depth__coverage__regions,depth__coverage__dist,depth__coverage__thresholds,align_bam -- sentinel_inputs=postprocess_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1033 - ramMin: 4096 - tmpdirMin: 5 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon -- class: arv:APIRequirement -inputs: -- id: postprocess_alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record -outputs: -- id: config__algorithm__coverage_interval - type: - - string - - 'null' -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -- id: regions__callable - type: - - File - - 'null' -- id: regions__sample_callable - type: - - File - - 'null' -- id: regions__nblock - type: - - File - - 'null' -- id: depth__samtools__stats - type: - - File - - 'null' -- id: depth__samtools__idxstats - type: - - File - - 'null' -- id: depth__variant_regions__regions - type: - - File - - 'null' -- id: depth__variant_regions__dist - type: - - File - - 'null' -- id: depth__sv_regions__regions - type: - - File - - 'null' -- id: depth__sv_regions__dist - type: - - File - - 'null' -- id: depth__coverage__regions - type: - - File - - 'null' -- id: depth__coverage__dist - type: - - File - - 'null' -- id: depth__coverage__thresholds - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_alignment_to_rec.cwl deleted file mode 100644 index 813f3fed737..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_alignment_to_rec.cwl +++ /dev/null @@ -1,233 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=postprocess_alignment_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;genome_resources__variation__lcr;config__algorithm__recalibrate;config__algorithm__coverage;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__tools_on;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__variant_regions_orig;config__algorithm__coverage_merged;config__algorithm__coverage_orig;config__algorithm__seq2c_bed_ready -- sentinel_inputs=align_bam:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__variant_regions_orig:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,config__algorithm__coverage_orig:var,config__algorithm__seq2c_bed_ready:var,config__algorithm__recalibrate:var,config__algorithm__tools_on:var,genome_resources__rnaseq__gene_bed:var,genome_resources__variation__dbsnp:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__seq2c_bed_ready - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: postprocess_alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_variants.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_variants.cwl deleted file mode 100644 index a300aa5be18..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/postprocess_variants.cwl +++ /dev/null @@ -1,195 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vrn_file_joint -- sentinel_inputs=jointvc_batch_rec:record,vrn_file_joint:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_variants -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 4096 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: snpeff - specs: - - https://anaconda.org/bioconda/snpeff - version: - - 4.3.1t -inputs: -- id: jointvc_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: jointvc_batch_rec - type: record - type: array -- id: vrn_file_joint_toolinput - secondaryFiles: - - .tbi - type: File -outputs: -- id: vrn_file_joint - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_align_inputs.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_align_inputs.cwl deleted file mode 100644 index 8179e185e8b..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_align_inputs.cwl +++ /dev/null @@ -1,142 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-split -- sentinel_outputs=process_alignment_rec:files;config__algorithm__quality_format;align_split -- sentinel_inputs=alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_align_inputs -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 4096 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: atropos;env - specs: - - https://anaconda.org/bioconda/atropos;env - version: - - python3 - - package: optitype - specs: - - https://anaconda.org/bioconda/optitype - - package: razers3 - specs: - - https://anaconda.org/bioconda/razers3 - version: - - 3.5.0 - - package: coincbc - specs: - - https://anaconda.org/bioconda/coincbc -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -outputs: -- id: process_alignment_rec - type: - items: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_samples.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_samples.cwl deleted file mode 100644 index 244502016fb..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_samples.cwl +++ /dev/null @@ -1,95 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=rgnames__sample,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready -- sentinel_inputs=prep_samples_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 2048 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy -inputs: -- id: prep_samples_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record -outputs: -- id: rgnames__sample - type: string -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_samples_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_samples_to_rec.cwl deleted file mode 100644 index 830e65f0404..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/prep_samples_to_rec.cwl +++ /dev/null @@ -1,85 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=prep_samples_rec:resources;description;reference__fasta__base;config__algorithm__coverage;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=config__algorithm__coverage:var,rgnames__sample:var,config__algorithm__variant_regions:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 2048 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: prep_samples_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/process_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/process_alignment.cwl deleted file mode 100644 index b12a9946b5c..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/process_alignment.cwl +++ /dev/null @@ -1,198 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-parallel -- sentinel_outputs=work_bam,align_bam,hla__fastq,work_bam_plus__disc,work_bam_plus__sr -- sentinel_inputs=alignment_rec:record,process_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- process_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 4096 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 7 -- class: SoftwareRequirement - packages: - - package: bwa - specs: - - https://anaconda.org/bioconda/bwa - - package: bwakit - specs: - - https://anaconda.org/bioconda/bwakit - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: minimap2 - specs: - - https://anaconda.org/bioconda/minimap2 - - package: novoalign - specs: - - https://anaconda.org/bioconda/novoalign - - package: snap-aligner - specs: - - https://anaconda.org/bioconda/snap-aligner - version: - - 1.0dev.97 - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: fgbio - specs: - - https://anaconda.org/bioconda/fgbio - - package: umis - specs: - - https://anaconda.org/bioconda/umis - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: seqtk - specs: - - https://anaconda.org/bioconda/seqtk - - package: samblaster - specs: - - https://anaconda.org/bioconda/samblaster - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -- class: arv:APIRequirement -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -- id: process_alignment_rec - type: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record -outputs: -- id: work_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/qc_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/qc_to_rec.cwl deleted file mode 100644 index 53e0a326097..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/qc_to_rec.cwl +++ /dev/null @@ -1,295 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=qc_rec:resources;description;reference__fasta__base;metadata__phenotype;config__algorithm__coverage_interval;metadata__batch;reference__versions;genome_build;config__algorithm__coverage;config__algorithm__tools_off;config__algorithm__qc;analysis;config__algorithm__tools_on;config__algorithm__variant_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__coverage_merged;depth__samtools__stats;depth__samtools__idxstats;depth__variant_regions__regions;depth__variant_regions__dist;depth__sv_regions__regions;depth__sv_regions__dist;depth__coverage__regions;depth__coverage__dist;depth__coverage__thresholds;variants__samples -- sentinel_inputs=align_bam:var,analysis:var,reference__fasta__base:var,reference__versions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,genome_build:var,config__algorithm__qc:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__coverage_interval:var,depth__variant_regions__regions:var,depth__variant_regions__dist:var,depth__samtools__stats:var,depth__samtools__idxstats:var,depth__sv_regions__regions:var,depth__sv_regions__dist:var,depth__coverage__regions:var,depth__coverage__dist:var,depth__coverage__thresholds:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,variants__samples:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- qc_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: analysis - type: - items: string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__versions - type: - items: File - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_build - type: - items: string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__variant_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__stats - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__idxstats - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__regions - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__thresholds - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: qc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/run_jointvc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/run_jointvc.cwl deleted file mode 100644 index 897c1e02ca8..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/run_jointvc.cwl +++ /dev/null @@ -1,203 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-parallel -- sentinel_outputs=vrn_file_region,region -- sentinel_inputs=jointvc_batch_rec:record,region:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- run_jointvc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: gvcfgenotyper - specs: - - https://anaconda.org/bioconda/gvcfgenotyper - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon -- class: arv:APIRequirement -inputs: -- id: jointvc_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: jointvc_batch_rec - type: record - type: array -- id: region_toolinput - type: string -outputs: -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - - File - - 'null' -- id: region - type: string -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/summarize_vc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/summarize_vc.cwl deleted file mode 100644 index 85b8a8f01e0..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/summarize_vc.cwl +++ /dev/null @@ -1,225 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=variants__calls,variants__gvcf,variants__samples,validate__grading_summary,validate__grading_plots -- sentinel_inputs=jointvc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_vc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 2048 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: jointvc_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - - name: vrn_file_joint - type: File - name: jointvc_rec - type: record - type: array - type: array -outputs: -- id: variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: validate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/variantcall_batch_region.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/variantcall_batch_region.cwl deleted file mode 100644 index eb58aa47177..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/steps/variantcall_batch_region.cwl +++ /dev/null @@ -1,266 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-parallel -- sentinel_outputs=vrn_file_region,region_block -- sentinel_inputs=batch_rec:record,region_block:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- variantcall_batch_region -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 4096 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: freebayes - specs: - - https://anaconda.org/bioconda/freebayes - version: - - 1.1.0.46 - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: vqsr_cnn - specs: - - https://anaconda.org/bioconda/vqsr_cnn - - package: deepvariant - specs: - - https://anaconda.org/bioconda/deepvariant - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: octopus - specs: - - https://anaconda.org/bioconda/octopus - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: platypus-variant - specs: - - https://anaconda.org/bioconda/platypus-variant - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: strelka - specs: - - https://anaconda.org/bioconda/strelka - - package: vardict - specs: - - https://anaconda.org/bioconda/vardict - - package: vardict-java - specs: - - https://anaconda.org/bioconda/vardict-java - - package: varscan - specs: - - https://anaconda.org/bioconda/varscan - - package: moreutils - specs: - - https://anaconda.org/bioconda/moreutils - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno - - package: vcflib - specs: - - https://anaconda.org/bioconda/vcflib - - package: vt - specs: - - https://anaconda.org/bioconda/vt - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: perl - specs: - - https://anaconda.org/bioconda/perl -- class: arv:APIRequirement -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block_toolinput - type: - items: string - type: array -outputs: -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - - File - - 'null' -- id: region_block - type: - items: string - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-alignment.cwl deleted file mode 100644 index 27c76c747c5..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-alignment.cwl +++ /dev/null @@ -1,143 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -outputs: -- id: align_bam - outputSource: merge_split_alignments/align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - outputSource: merge_split_alignments/work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - outputSource: merge_split_alignments/work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - outputSource: merge_split_alignments/hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: prep_align_inputs - in: - - id: alignment_rec - source: alignment_rec - out: - - id: process_alignment_rec - run: steps/prep_align_inputs.cwl -- id: process_alignment - in: - - id: alignment_rec - source: alignment_rec - - id: process_alignment_rec - source: prep_align_inputs/process_alignment_rec - out: - - id: work_bam - - id: align_bam - - id: hla__fastq - - id: work_bam_plus__disc - - id: work_bam_plus__sr - run: steps/process_alignment.cwl - scatter: - - process_alignment_rec - scatterMethod: dotproduct -- id: merge_split_alignments - in: - - id: alignment_rec - source: alignment_rec - - id: work_bam - source: process_alignment/work_bam - - id: align_bam_toolinput - source: process_alignment/align_bam - - id: work_bam_plus__disc_toolinput - source: process_alignment/work_bam_plus__disc - - id: work_bam_plus__sr_toolinput - source: process_alignment/work_bam_plus__sr - - id: hla__fastq_toolinput - source: process_alignment/hla__fastq - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: steps/merge_split_alignments.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-jointcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-jointcall.cwl deleted file mode 100644 index 3091d1de370..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-jointcall.cwl +++ /dev/null @@ -1,360 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: jointvc_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: jointvc_batch_rec - type: record - type: array -outputs: -- id: jointvc_rec - outputSource: finalize_jointvc/jointvc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - - name: vrn_file_joint - type: File - name: jointvc_rec - type: record - type: array -- id: vrn_file_joint - outputSource: postprocess_variants/vrn_file_joint - secondaryFiles: - - .tbi - type: File -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: get_parallel_regions_jointvc - in: - - id: jointvc_batch_rec - source: jointvc_batch_rec - out: - - id: region - run: steps/get_parallel_regions_jointvc.cwl -- id: run_jointvc - in: - - id: jointvc_batch_rec - source: jointvc_batch_rec - - id: region_toolinput - source: get_parallel_regions_jointvc/region - out: - - id: vrn_file_region - - id: region - run: steps/run_jointvc.cwl - scatter: - - region_toolinput - scatterMethod: dotproduct -- id: concat_batch_variantcalls_jointvc - in: - - id: jointvc_batch_rec - source: jointvc_batch_rec - - id: region - source: run_jointvc/region - - id: vrn_file_region - source: run_jointvc/vrn_file_region - out: - - id: vrn_file_joint - run: steps/concat_batch_variantcalls_jointvc.cwl -- id: postprocess_variants - in: - - id: jointvc_batch_rec - source: jointvc_batch_rec - - id: vrn_file_joint_toolinput - source: concat_batch_variantcalls_jointvc/vrn_file_joint - out: - - id: vrn_file_joint - run: steps/postprocess_variants.cwl -- id: finalize_jointvc - in: - - id: jointvc_batch_rec - source: jointvc_batch_rec - - id: vrn_file_joint - source: postprocess_variants/vrn_file_joint - out: - - id: jointvc_rec - run: steps/finalize_jointvc.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-variantcall.cwl deleted file mode 100644 index ffabd7e9f1e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/gvcf-joint-workflow/wf-variantcall.cwl +++ /dev/null @@ -1,329 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: vc_rec - outputSource: compare_to_rm/vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - File - - 'null' - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: get_parallel_regions - in: - - id: batch_rec - source: batch_rec - out: - - id: region_block - run: steps/get_parallel_regions.cwl -- id: variantcall_batch_region - in: - - id: batch_rec - source: batch_rec - - id: region_block_toolinput - source: get_parallel_regions/region_block - out: - - id: vrn_file_region - - id: region_block - run: steps/variantcall_batch_region.cwl - scatter: - - region_block_toolinput - scatterMethod: dotproduct -- id: concat_batch_variantcalls - in: - - id: batch_rec - source: batch_rec - - id: region_block - source: variantcall_batch_region/region_block - - id: vrn_file_region - source: variantcall_batch_region/vrn_file_region - out: - - id: vrn_file - run: steps/concat_batch_variantcalls.cwl -- id: compare_to_rm - in: - - id: batch_rec - source: batch_rec - - id: vrn_file - source: concat_batch_variantcalls/vrn_file - out: - - id: vc_rec - run: steps/compare_to_rm.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/main-prealign-samples.json b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/main-prealign-samples.json deleted file mode 100644 index 3e1234addda..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/main-prealign-samples.json +++ /dev/null @@ -1,526 +0,0 @@ -{ - "analysis": [ - "variant2", - "variant2" - ], - "config__algorithm__coverage": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/coverage_transcripts-bam.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/coverage_transcripts-bam.bed" - } - ], - "config__algorithm__coverage_interval": [ - null, - null - ], - "config__algorithm__effects": [ - "snpeff", - "snpeff" - ], - "config__algorithm__ensemble": [ - null, - null - ], - "config__algorithm__exclude_regions": [ - [], - [] - ], - "config__algorithm__min_allele_fraction": [ - 10.0, - 10.0 - ], - "config__algorithm__nomap_split_size": [ - 250, - 250 - ], - "config__algorithm__nomap_split_targets": [ - 20, - 20 - ], - "config__algorithm__qc": [ - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "qualimap", - "samtools", - "variants" - ], - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "qualimap", - "samtools", - "variants" - ] - ], - "config__algorithm__recalibrate": [ - "False", - "False" - ], - "config__algorithm__tools_off": [ - [], - [] - ], - "config__algorithm__tools_on": [ - [ - "qualimap_full" - ], - [ - "qualimap_full" - ] - ], - "config__algorithm__validate": [ - null, - null - ], - "config__algorithm__validate_regions": [ - null, - null - ], - "config__algorithm__variant_regions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - } - ], - "config__algorithm__variantcaller": [ - "False", - "False" - ], - "config__algorithm__vcfanno": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.conf" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.lua" - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.conf" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.lua" - } - ] - ], - "description": [ - "Test1", - "Test2" - ], - "files": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/6_100326_FC6107FAAXX.bam" - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/6_100326_FC6107FAAXX_2.bam" - } - ] - ], - "genome_build": [ - "hg19", - "hg19" - ], - "genome_resources__aliases__ensembl": [ - "homo_sapiens_vep_83_GRCh37", - "homo_sapiens_vep_83_GRCh37" - ], - "genome_resources__aliases__human": [ - "True", - "True" - ], - "genome_resources__aliases__snpeff": [ - "hg19", - "hg19" - ], - "genome_resources__rnaseq__gene_bed": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - } - ], - "genome_resources__variation__1000g": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__clinvar": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__cosmic": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__dbsnp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__encode_blacklist": [ - null, - null - ], - "genome_resources__variation__esp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__exac": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__gnomad_exome": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__lcr": [ - null, - null - ], - "genome_resources__variation__polyx": [ - null, - null - ], - "genome_resources__variation__train_hapmap": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__train_indels": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - } - ], - "metadata__batch": [ - null, - null - ], - "metadata__phenotype": [ - "", - "" - ], - "reference__fasta__base": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - } - ], - "reference__genome_context": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ] - ], - "reference__rtg": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - } - ], - "reference__snpeff__hg19": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - } - ], - "reference__versions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - } - ], - "resources": [ - "{}", - "{}" - ], - "rgnames__sample": [ - "Test1", - "Test2" - ], - "vrn_file": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/Test1-gatk-haplotype.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/Test1-gatk-haplotype.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/Test2-gatk-haplotype.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/Test2-gatk-haplotype.vcf.gz.tbi" - } - ] - } - ] -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/main-prealign.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/main-prealign.cwl deleted file mode 100644 index 535082448ed..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/main-prealign.cwl +++ /dev/null @@ -1,672 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: files - type: - items: - items: File - type: array - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__vcfanno - type: - items: - items: File - type: array - type: array -- id: resources - type: - items: string - type: array -- id: config__algorithm__variantcaller - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: metadata__batch - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__versions - type: - items: File - type: array -- id: vrn_file - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__validate - type: - items: - - 'null' - - string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: description - type: - items: string - type: array -- id: config__algorithm__validate_regions - type: - items: - - 'null' - - string - type: array -- id: genome_build - type: - items: string - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__coverage - type: - items: File - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__rtg - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -outputs: -- id: rgnames__sample_out - outputSource: prep_samples/rgnames__sample - type: - items: string - type: array -- id: align_bam - outputSource: postprocess_alignment/align_bam - type: - items: - - File - - 'null' - type: array -- id: regions__sample_callable - outputSource: postprocess_alignment/regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: validate__grading_summary - outputSource: summarize_vc/validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: variants__calls - outputSource: summarize_vc/variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - outputSource: summarize_vc/variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: summary__multiqc - outputSource: multiqc_summary/summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - outputSource: multiqc_summary/versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - outputSource: multiqc_summary/versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: organize_noalign - in: - - id: files - source: files - - id: resources - source: resources - - id: description - source: description - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: steps/organize_noalign.cwl - scatter: - - files - - resources - - description - scatterMethod: dotproduct -- id: prep_samples_to_rec - in: - - id: config__algorithm__coverage - source: config__algorithm__coverage - - id: rgnames__sample - source: rgnames__sample - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: prep_samples_rec - run: steps/prep_samples_to_rec.cwl -- id: prep_samples - in: - - id: prep_samples_rec - source: prep_samples_to_rec/prep_samples_rec - out: - - id: rgnames__sample - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - run: steps/prep_samples.cwl - scatter: - - prep_samples_rec - scatterMethod: dotproduct -- id: postprocess_alignment_to_rec - in: - - id: align_bam - source: organize_noalign/align_bam - - id: config__algorithm__coverage_interval - source: config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: prep_samples/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: prep_samples/config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - source: prep_samples/config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - source: prep_samples/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: prep_samples/config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - source: prep_samples/config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - source: prep_samples/config__algorithm__seq2c_bed_ready - - id: config__algorithm__recalibrate - source: config__algorithm__recalibrate - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: postprocess_alignment_rec - run: steps/postprocess_alignment_to_rec.cwl -- id: postprocess_alignment - in: - - id: postprocess_alignment_rec - source: postprocess_alignment_to_rec/postprocess_alignment_rec - out: - - id: config__algorithm__coverage_interval - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - - id: regions__callable - - id: regions__sample_callable - - id: regions__nblock - - id: depth__samtools__stats - - id: depth__samtools__idxstats - - id: depth__variant_regions__regions - - id: depth__variant_regions__dist - - id: depth__sv_regions__regions - - id: depth__sv_regions__dist - - id: depth__coverage__regions - - id: depth__coverage__dist - - id: depth__coverage__thresholds - - id: align_bam - run: steps/postprocess_alignment.cwl - scatter: - - postprocess_alignment_rec - scatterMethod: dotproduct -- id: combine_sample_regions - in: - - id: regions__callable - source: postprocess_alignment/regions__callable - - id: regions__nblock - source: postprocess_alignment/regions__nblock - - id: metadata__batch - source: metadata__batch - - id: config__algorithm__nomap_split_size - source: config__algorithm__nomap_split_size - - id: config__algorithm__nomap_split_targets - source: config__algorithm__nomap_split_targets - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: config__algorithm__callable_regions - - id: config__algorithm__non_callable_regions - - id: config__algorithm__callable_count - run: steps/combine_sample_regions.cwl -- id: batch_for_variantcall - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: align_bam - source: postprocess_alignment/align_bam - - id: vrn_file - source: vrn_file - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: config__algorithm__variantcaller - source: config__algorithm__variantcaller - - id: config__algorithm__ensemble - source: config__algorithm__ensemble - - id: config__algorithm__vcfanno - source: config__algorithm__vcfanno - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__effects - source: config__algorithm__effects - - id: config__algorithm__min_allele_fraction - source: config__algorithm__min_allele_fraction - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__validate - source: config__algorithm__validate - - id: config__algorithm__validate_regions - source: config__algorithm__validate_regions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__rtg - source: reference__rtg - - id: reference__genome_context - source: reference__genome_context - - id: genome_resources__variation__clinvar - source: genome_resources__variation__clinvar - - id: genome_resources__variation__cosmic - source: genome_resources__variation__cosmic - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__esp - source: genome_resources__variation__esp - - id: genome_resources__variation__exac - source: genome_resources__variation__exac - - id: genome_resources__variation__gnomad_exome - source: genome_resources__variation__gnomad_exome - - id: genome_resources__variation__1000g - source: genome_resources__variation__1000g - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__aliases__ensembl - source: genome_resources__aliases__ensembl - - id: genome_resources__aliases__human - source: genome_resources__aliases__human - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__hg19 - source: reference__snpeff__hg19 - - id: genome_resources__variation__train_hapmap - source: genome_resources__variation__train_hapmap - - id: genome_resources__variation__train_indels - source: genome_resources__variation__train_indels - - id: resources - source: resources - - id: description - source: description - out: - - id: batch_rec - run: steps/batch_for_variantcall.cwl -- id: variantcall - in: - - id: batch_rec - source: batch_for_variantcall/batch_rec - out: - - id: vc_rec - run: wf-variantcall.cwl - scatter: - - batch_rec - scatterMethod: dotproduct -- id: summarize_vc - in: - - id: vc_rec - source: variantcall/vc_rec - out: - - id: variants__calls - - id: variants__gvcf - - id: variants__samples - - id: validate__grading_summary - - id: validate__grading_plots - run: steps/summarize_vc.cwl -- id: qc_to_rec - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: analysis - source: analysis - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__versions - source: reference__versions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: genome_build - source: genome_build - - id: config__algorithm__qc - source: config__algorithm__qc - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: depth__variant_regions__dist - source: postprocess_alignment/depth__variant_regions__dist - - id: depth__samtools__stats - source: postprocess_alignment/depth__samtools__stats - - id: depth__samtools__idxstats - source: postprocess_alignment/depth__samtools__idxstats - - id: depth__sv_regions__regions - source: postprocess_alignment/depth__sv_regions__regions - - id: depth__sv_regions__dist - source: postprocess_alignment/depth__sv_regions__dist - - id: depth__coverage__regions - source: postprocess_alignment/depth__coverage__regions - - id: depth__coverage__dist - source: postprocess_alignment/depth__coverage__dist - - id: depth__coverage__thresholds - source: postprocess_alignment/depth__coverage__thresholds - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__coverage - source: postprocess_alignment/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: postprocess_alignment/config__algorithm__coverage_merged - - id: variants__samples - source: summarize_vc/variants__samples - - id: resources - source: resources - - id: description - source: description - out: - - id: qc_rec - run: steps/qc_to_rec.cwl -- id: pipeline_summary - in: - - id: qc_rec - source: qc_to_rec/qc_rec - out: - - id: qcout_rec - run: steps/pipeline_summary.cwl - scatter: - - qc_rec - scatterMethod: dotproduct -- id: multiqc_summary - in: - - id: qcout_rec - source: pipeline_summary/qcout_rec - out: - - id: summary__multiqc - - id: versions__tools - - id: versions__data - run: steps/multiqc_summary.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/batch_for_variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/batch_for_variantcall.cwl deleted file mode 100644 index 1ad9465444b..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/batch_for_variantcall.cwl +++ /dev/null @@ -1,407 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=batch_rec:resources;description;reference__fasta__base;metadata__phenotype;config__algorithm__vcfanno;config__algorithm__variantcaller;genome_resources__variation__1000g;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;config__algorithm__min_allele_fraction;vrn_file;genome_resources__variation__train_hapmap;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;align_bam;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,align_bam:var,vrn_file:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,regions__sample_callable:var,config__algorithm__variantcaller:var,config__algorithm__ensemble:var,config__algorithm__vcfanno:var,config__algorithm__coverage_interval:var,config__algorithm__effects:var,config__algorithm__min_allele_fraction:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__validate:var,config__algorithm__validate_regions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,reference__fasta__base:var,reference__rtg:var,reference__genome_context:var,genome_resources__variation__clinvar:var,genome_resources__variation__cosmic:var,genome_resources__variation__dbsnp:var,genome_resources__variation__esp:var,genome_resources__variation__exac:var,genome_resources__variation__gnomad_exome:var,genome_resources__variation__1000g:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__aliases__ensembl:var,genome_resources__aliases__human:var,genome_resources__aliases__snpeff:var,reference__snpeff__hg19:var,genome_resources__variation__train_hapmap:var,genome_resources__variation__train_indels:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_variantcall -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: vrn_file - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: metadata__batch - type: - items: - - 'null' - - string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variantcaller - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__vcfanno - type: - items: - items: File - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__validate - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__validate_regions - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__rtg - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: - - 'null' - - string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: File - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/combine_sample_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/combine_sample_regions.cwl deleted file mode 100644 index 2df313f7a8f..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/combine_sample_regions.cwl +++ /dev/null @@ -1,101 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=config__algorithm__callable_regions,config__algorithm__non_callable_regions,config__algorithm__callable_count -- sentinel_inputs=regions__callable:var,regions__nblock:var,metadata__batch:var,config__algorithm__nomap_split_size:var,config__algorithm__nomap_split_targets:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- combine_sample_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: regions__callable - type: - items: - - File - - 'null' - type: array -- id: regions__nblock - type: - items: - - File - - 'null' - type: array -- id: metadata__batch - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__non_callable_regions - type: - items: File - type: array -- id: config__algorithm__callable_count - type: - items: int - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/compare_to_rm.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/compare_to_rm.cwl deleted file mode 100644 index ab4909659ca..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/compare_to_rm.cwl +++ /dev/null @@ -1,315 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vc_rec:batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;resources;description;vrn_file;reference__fasta__base;metadata__phenotype;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;metadata__batch;config__algorithm__min_allele_fraction;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__aliases__human;config__algorithm__tools_off;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- compare_to_rm -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: gvcf-regions - specs: - - https://anaconda.org/bioconda/gvcf-regions - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: rtg-tools - specs: - - https://anaconda.org/bioconda/rtg-tools - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: - - 'null' - - string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: File - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file - secondaryFiles: - - .tbi - type: File -outputs: -- id: vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/concat_batch_variantcalls.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/concat_batch_variantcalls.cwl deleted file mode 100644 index ae12d21651d..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/concat_batch_variantcalls.cwl +++ /dev/null @@ -1,199 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-merge -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,region_block:var,vrn_file_region:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- concat_batch_variantcalls -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: - - 'null' - - string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: File - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block - type: - items: - items: string - type: array - type: array -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/get_parallel_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/get_parallel_regions.cwl deleted file mode 100644 index c4e3997bca4..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/get_parallel_regions.cwl +++ /dev/null @@ -1,174 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-split -- sentinel_outputs=region_block -- sentinel_inputs=batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- get_parallel_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: - - 'null' - - string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: File - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: region_block - type: - items: - items: string - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/multiqc_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/multiqc_summary.cwl deleted file mode 100644 index 82ce9f97a43..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/multiqc_summary.cwl +++ /dev/null @@ -1,95 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=summary__multiqc,versions__tools,versions__data -- sentinel_inputs=qcout_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- multiqc_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: multiqc - specs: - - https://anaconda.org/bioconda/multiqc - - package: multiqc-bcbio - specs: - - https://anaconda.org/bioconda/multiqc-bcbio -inputs: -- id: qcout_rec - type: - items: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record - type: array -outputs: -- id: summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/organize_noalign.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/organize_noalign.cwl deleted file mode 100644 index 370f2ccf982..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/organize_noalign.cwl +++ /dev/null @@ -1,59 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=align_bam,work_bam_plus__disc,work_bam_plus__sr,hla__fastq -- sentinel_inputs=files:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- organize_noalign -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1024 - ramMin: 3072 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 4 -inputs: -- id: files - type: - items: File - type: array -- id: resources - type: string -- id: description - type: string -outputs: -- id: align_bam - secondaryFiles: - - .bai - type: File -- id: work_bam_plus__disc - type: - - File - - 'null' -- id: work_bam_plus__sr - type: - - File - - 'null' -- id: hla__fastq - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/pipeline_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/pipeline_summary.cwl deleted file mode 100644 index c0e41e5ba62..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/pipeline_summary.cwl +++ /dev/null @@ -1,221 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=qcout_rec:summary__qc;summary__metrics;description;reference__versions;genome_build;config__algorithm__tools_off;config__algorithm__qc;config__algorithm__tools_on -- sentinel_inputs=qc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- pipeline_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: fastqc=0.11.7=5 - specs: - - https://anaconda.org/bioconda/fastqc=0.11.7=5 - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: hts-nim-tools - specs: - - https://anaconda.org/bioconda/hts-nim-tools - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: qsignature - specs: - - https://anaconda.org/bioconda/qsignature - - package: qualimap - specs: - - https://anaconda.org/bioconda/qualimap - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: preseq - specs: - - https://anaconda.org/bioconda/preseq - - package: peddy - specs: - - https://anaconda.org/bioconda/peddy - - package: verifybamid2 - specs: - - https://anaconda.org/bioconda/verifybamid2 -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: qc_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: - - 'null' - - string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record -outputs: -- id: qcout_rec - type: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_alignment.cwl deleted file mode 100644 index 6a261c205af..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_alignment.cwl +++ /dev/null @@ -1,221 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=config__algorithm__coverage_interval,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready,regions__callable,regions__sample_callable,regions__nblock,depth__samtools__stats,depth__samtools__idxstats,depth__variant_regions__regions,depth__variant_regions__dist,depth__sv_regions__regions,depth__sv_regions__dist,depth__coverage__regions,depth__coverage__dist,depth__coverage__thresholds,align_bam -- sentinel_inputs=postprocess_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1033 - ramMin: 6144 - tmpdirMin: 5 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon -- class: arv:APIRequirement -inputs: -- id: postprocess_alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: File - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record -outputs: -- id: config__algorithm__coverage_interval - type: - - string - - 'null' -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -- id: regions__callable - type: - - File - - 'null' -- id: regions__sample_callable - type: - - File - - 'null' -- id: regions__nblock - type: - - File - - 'null' -- id: depth__samtools__stats - type: - - File - - 'null' -- id: depth__samtools__idxstats - type: - - File - - 'null' -- id: depth__variant_regions__regions - type: - - File - - 'null' -- id: depth__variant_regions__dist - type: - - File - - 'null' -- id: depth__sv_regions__regions - type: - - File - - 'null' -- id: depth__sv_regions__dist - type: - - File - - 'null' -- id: depth__coverage__regions - type: - - File - - 'null' -- id: depth__coverage__dist - type: - - File - - 'null' -- id: depth__coverage__thresholds - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_alignment_to_rec.cwl deleted file mode 100644 index e61b71f4d28..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_alignment_to_rec.cwl +++ /dev/null @@ -1,229 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=postprocess_alignment_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;genome_resources__variation__lcr;config__algorithm__recalibrate;config__algorithm__coverage;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__tools_on;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__variant_regions_orig;config__algorithm__coverage_merged;config__algorithm__coverage_orig;config__algorithm__seq2c_bed_ready -- sentinel_inputs=align_bam:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__variant_regions_orig:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,config__algorithm__coverage_orig:var,config__algorithm__seq2c_bed_ready:var,config__algorithm__recalibrate:var,config__algorithm__tools_on:var,genome_resources__rnaseq__gene_bed:var,genome_resources__variation__dbsnp:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: File - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__seq2c_bed_ready - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: postprocess_alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: File - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_variants.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_variants.cwl deleted file mode 100644 index dd7e67c7d61..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/postprocess_variants.cwl +++ /dev/null @@ -1,183 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_variants -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1025 - ramMin: 6144 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: snpeff - specs: - - https://anaconda.org/bioconda/snpeff - version: - - 4.3.1t -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: - - 'null' - - string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: File - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file_toolinput - secondaryFiles: - - .tbi - type: File -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/prep_samples.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/prep_samples.cwl deleted file mode 100644 index f473b969345..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/prep_samples.cwl +++ /dev/null @@ -1,93 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=rgnames__sample,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready -- sentinel_inputs=prep_samples_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy -inputs: -- id: prep_samples_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage - type: File - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record -outputs: -- id: rgnames__sample - type: string -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/prep_samples_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/prep_samples_to_rec.cwl deleted file mode 100644 index 54f7781bb21..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/prep_samples_to_rec.cwl +++ /dev/null @@ -1,81 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=prep_samples_rec:resources;description;reference__fasta__base;config__algorithm__coverage;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=config__algorithm__coverage:var,rgnames__sample:var,config__algorithm__variant_regions:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: config__algorithm__coverage - type: - items: File - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: prep_samples_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage - type: File - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/qc_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/qc_to_rec.cwl deleted file mode 100644 index 3de03a1a1a8..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/qc_to_rec.cwl +++ /dev/null @@ -1,299 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=qc_rec:resources;description;reference__fasta__base;metadata__phenotype;config__algorithm__coverage_interval;metadata__batch;reference__versions;genome_build;config__algorithm__coverage;config__algorithm__tools_off;config__algorithm__qc;analysis;config__algorithm__tools_on;config__algorithm__variant_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__coverage_merged;depth__samtools__stats;depth__samtools__idxstats;depth__variant_regions__regions;depth__variant_regions__dist;depth__sv_regions__regions;depth__sv_regions__dist;depth__coverage__regions;depth__coverage__dist;depth__coverage__thresholds;variants__samples -- sentinel_inputs=align_bam:var,analysis:var,reference__fasta__base:var,reference__versions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,genome_build:var,config__algorithm__qc:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__coverage_interval:var,depth__variant_regions__regions:var,depth__variant_regions__dist:var,depth__samtools__stats:var,depth__samtools__idxstats:var,depth__sv_regions__regions:var,depth__sv_regions__dist:var,depth__coverage__regions:var,depth__coverage__dist:var,depth__coverage__thresholds:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,variants__samples:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- qc_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: analysis - type: - items: string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__versions - type: - items: File - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_build - type: - items: string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: metadata__batch - type: - items: - - 'null' - - string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__variant_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__stats - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__idxstats - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__regions - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__thresholds - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: qc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: - - 'null' - - string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/summarize_vc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/summarize_vc.cwl deleted file mode 100644 index e9fde20378c..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/summarize_vc.cwl +++ /dev/null @@ -1,198 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=variants__calls,variants__gvcf,variants__samples,validate__grading_summary,validate__grading_plots -- sentinel_inputs=vc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_vc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: vc_rec - type: - items: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array - type: array -outputs: -- id: variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: validate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/variantcall_batch_region.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/variantcall_batch_region.cwl deleted file mode 100644 index e9192828e62..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/steps/variantcall_batch_region.cwl +++ /dev/null @@ -1,269 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-parallel -- sentinel_outputs=vrn_file_region,region_block -- sentinel_inputs=batch_rec:record,region_block:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- variantcall_batch_region -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: freebayes - specs: - - https://anaconda.org/bioconda/freebayes - version: - - 1.1.0.46 - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: vqsr_cnn - specs: - - https://anaconda.org/bioconda/vqsr_cnn - - package: deepvariant - specs: - - https://anaconda.org/bioconda/deepvariant - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: octopus - specs: - - https://anaconda.org/bioconda/octopus - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: platypus-variant - specs: - - https://anaconda.org/bioconda/platypus-variant - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: strelka - specs: - - https://anaconda.org/bioconda/strelka - - package: vardict - specs: - - https://anaconda.org/bioconda/vardict - - package: vardict-java - specs: - - https://anaconda.org/bioconda/vardict-java - - package: varscan - specs: - - https://anaconda.org/bioconda/varscan - - package: moreutils - specs: - - https://anaconda.org/bioconda/moreutils - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno - - package: vcflib - specs: - - https://anaconda.org/bioconda/vcflib - - package: vt - specs: - - https://anaconda.org/bioconda/vt - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: perl - specs: - - https://anaconda.org/bioconda/perl -- class: arv:APIRequirement -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: - - 'null' - - string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: File - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block_toolinput - type: - items: string - type: array -outputs: -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - - File - - 'null' -- id: region_block - type: - items: string - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/wf-variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/wf-variantcall.cwl deleted file mode 100644 index b48a8254984..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/prealign-workflow/wf-variantcall.cwl +++ /dev/null @@ -1,316 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: - - 'null' - - string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: File - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: vc_rec - outputSource: compare_to_rm/vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: metadata__phenotype - type: string - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: - - boolean - - 'null' - - string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: - - 'null' - - string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: get_parallel_regions - in: - - id: batch_rec - source: batch_rec - out: - - id: region_block - run: steps/get_parallel_regions.cwl -- id: variantcall_batch_region - in: - - id: batch_rec - source: batch_rec - - id: region_block_toolinput - source: get_parallel_regions/region_block - out: - - id: vrn_file_region - - id: region_block - run: steps/variantcall_batch_region.cwl - scatter: - - region_block_toolinput - scatterMethod: dotproduct -- id: concat_batch_variantcalls - in: - - id: batch_rec - source: batch_rec - - id: region_block - source: variantcall_batch_region/region_block - - id: vrn_file_region - source: variantcall_batch_region/vrn_file_region - out: - - id: vrn_file - run: steps/concat_batch_variantcalls.cwl -- id: postprocess_variants - in: - - id: batch_rec - source: batch_rec - - id: vrn_file_toolinput - source: concat_batch_variantcalls/vrn_file - out: - - id: vrn_file - run: steps/postprocess_variants.cwl -- id: compare_to_rm - in: - - id: batch_rec - source: batch_rec - - id: vrn_file - source: postprocess_variants/vrn_file - out: - - id: vc_rec - run: steps/compare_to_rm.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/main-rnaseq-samples.json b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/main-rnaseq-samples.json deleted file mode 100644 index 3e67d52727e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/main-rnaseq-samples.json +++ /dev/null @@ -1,111 +0,0 @@ -{ - "analysis": [ - "RNA-seq" - ], - "config__algorithm__aligner": [ - "hisat2" - ], - "config__algorithm__coverage_interval": [ - null - ], - "config__algorithm__expression_caller": [ - [ - "salmon", - "kallisto" - ] - ], - "config__algorithm__fusion_caller": [ - [ - "pizzly" - ] - ], - "config__algorithm__qc": [ - [ - "fastqc", - "qualimap_rnaseq", - "samtools" - ] - ], - "config__algorithm__quality_format": [ - "standard" - ], - "config__algorithm__tools_off": [ - [] - ], - "config__algorithm__tools_on": [ - [] - ], - "description": [ - "Test1" - ], - "files": [ - [ - { - "class": "File", - "path": "gs://bcbio-cromwell-dev-test/test_bcbio_cwl/testdata/test_fusion/fusion-1_1.fq.gz" - }, - { - "class": "File", - "path": "gs://bcbio-cromwell-dev-test/test_bcbio_cwl/testdata/test_fusion/fusion-1_2.fq.gz" - } - ] - ], - "genome_build": [ - "hg19" - ], - "genome_resources__rnaseq__transcripts": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.gtf", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.gtf.db" - } - ] - } - ], - "reference__fasta__base": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - } - ], - "reference__hisat2__indexes": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/hisat2-wf.tar.gz" - } - ], - "resources": [ - "{}" - ], - "rgnames__lane": [ - "Test1" - ], - "rgnames__lb": [ - null - ], - "rgnames__pl": [ - "illumina" - ], - "rgnames__pu": [ - "Test1" - ], - "rgnames__rg": [ - "Test1" - ], - "rgnames__sample": [ - "Test1" - ] -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/main-rnaseq.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/main-rnaseq.cwl deleted file mode 100644 index 2dd258ac786..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/main-rnaseq.cwl +++ /dev/null @@ -1,304 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: files - type: - items: - items: File - type: array - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: config__algorithm__expression_caller - type: - items: - items: string - type: array - type: array -- id: resources - type: - items: string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: reference__hisat2__indexes - type: - items: File - type: array -- id: config__algorithm__fusion_caller - type: - items: - items: string - type: array - type: array -- id: description - type: - items: string - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: genome_resources__rnaseq__transcripts - secondaryFiles: - - .db - type: - items: File - type: array -- id: config__algorithm__quality_format - type: - items: string - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__tools_on - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: rgnames__lane - type: - items: string - type: array -outputs: -- id: rgnames__sample_out - outputSource: rgnames__sample - type: - items: string - type: array -- id: align_bam - outputSource: process_alignment/align_bam - type: - items: File - type: array -- id: quant__tsv - outputSource: rnaseq_quantitate/quant__tsv - type: - items: File - type: array -- id: summary__multiqc - outputSource: multiqc_summary/summary__multiqc - type: - items: - - File - - 'null' - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: prepare_sample - in: - - id: files - source: files - - id: rgnames__sample - source: rgnames__sample - - id: reference__fasta__base - source: reference__fasta__base - - id: genome_build - source: genome_build - - id: genome_resources__rnaseq__transcripts - source: genome_resources__rnaseq__transcripts - - id: analysis - source: analysis - - id: rgnames__pl - source: rgnames__pl - - id: rgnames__pu - source: rgnames__pu - - id: rgnames__lane - source: rgnames__lane - - id: rgnames__rg - source: rgnames__rg - - id: rgnames__lb - source: rgnames__lb - - id: reference__hisat2__indexes - source: reference__hisat2__indexes - - id: config__algorithm__aligner - source: config__algorithm__aligner - - id: config__algorithm__expression_caller - source: config__algorithm__expression_caller - - id: config__algorithm__fusion_caller - source: config__algorithm__fusion_caller - - id: config__algorithm__quality_format - source: config__algorithm__quality_format - - id: resources - source: resources - - id: description - source: description - out: - - id: prep_rec - run: steps/prepare_sample.cwl - scatter: - - files - - rgnames__sample - - reference__fasta__base - - genome_build - - genome_resources__rnaseq__transcripts - - analysis - - rgnames__pl - - rgnames__pu - - rgnames__lane - - rgnames__rg - - rgnames__lb - - reference__hisat2__indexes - - config__algorithm__aligner - - config__algorithm__expression_caller - - config__algorithm__fusion_caller - - config__algorithm__quality_format - - resources - - description - scatterMethod: dotproduct -- id: trim_sample - in: - - id: prep_rec - source: prepare_sample/prep_rec - out: - - id: trim_rec - run: steps/trim_sample.cwl - scatter: - - prep_rec - scatterMethod: dotproduct -- id: process_alignment - in: - - id: trim_rec - source: trim_sample/trim_rec - out: - - id: align_bam - run: steps/process_alignment.cwl - scatter: - - trim_rec - scatterMethod: dotproduct -- id: rnaseq_quantitate - in: - - id: trim_rec - source: trim_sample/trim_rec - - id: align_bam - source: process_alignment/align_bam - out: - - id: count_file - - id: quant__tsv - - id: quant__hdf5 - - id: quant__fusion - run: steps/rnaseq_quantitate.cwl - scatter: - - trim_rec - - align_bam - scatterMethod: dotproduct -- id: qc_to_rec - in: - - id: align_bam - source: process_alignment/align_bam - - id: analysis - source: analysis - - id: reference__fasta__base - source: reference__fasta__base - - id: genome_resources__rnaseq__transcripts - source: genome_resources__rnaseq__transcripts - - id: genome_build - source: genome_build - - id: config__algorithm__coverage_interval - source: config__algorithm__coverage_interval - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: config__algorithm__qc - source: config__algorithm__qc - - id: resources - source: resources - - id: description - source: description - out: - - id: qc_rec - run: steps/qc_to_rec.cwl -- id: pipeline_summary - in: - - id: qc_rec - source: qc_to_rec/qc_rec - out: - - id: qcout_rec - run: steps/pipeline_summary.cwl - scatter: - - qc_rec - scatterMethod: dotproduct -- id: multiqc_summary - in: - - id: qcout_rec - source: pipeline_summary/qcout_rec - out: - - id: summary__multiqc - run: steps/multiqc_summary.cwl -- id: detect_fusions - in: - - id: quant__fusion - source: rnaseq_quantitate/quant__fusion - - id: quant__hdf5 - source: rnaseq_quantitate/quant__hdf5 - - id: trim_rec - source: trim_sample/trim_rec - out: - - id: fusion__fasta - - id: fusion__json - run: steps/detect_fusions.cwl - scatter: - - quant__fusion - - quant__hdf5 - - trim_rec - scatterMethod: dotproduct diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/detect_fusions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/detect_fusions.cwl deleted file mode 100644 index 8803c214f31..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/detect_fusions.cwl +++ /dev/null @@ -1,97 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=fusion__fasta,fusion__json -- sentinel_inputs=quant__fusion:var,quant__hdf5:var,trim_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- detect_fusions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1024 - ramMin: 4096 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: pizzly - specs: - - https://anaconda.org/bioconda/pizzly -inputs: -- id: quant__fusion - type: File -- id: quant__hdf5 - type: File -- id: trim_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: files - type: - items: File - type: array - - name: reference__fasta__base - type: File - - name: config__algorithm__expression_caller - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: reference__hisat2__indexes - type: File - - name: config__algorithm__fusion_caller - type: - items: string - type: array - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: genome_build - type: string - - name: rgnames__pu - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__quality_format - type: string - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: rgnames__lane - type: string - name: trim_rec - type: record -outputs: -- id: fusion__fasta - type: File -- id: fusion__json - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/multiqc_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/multiqc_summary.cwl deleted file mode 100644 index 604f9291df7..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/multiqc_summary.cwl +++ /dev/null @@ -1,96 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=summary__multiqc -- sentinel_inputs=qcout_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- multiqc_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1024 - ramMin: 4096 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: multiqc - specs: - - https://anaconda.org/bioconda/multiqc - - package: multiqc-bcbio - specs: - - https://anaconda.org/bioconda/multiqc-bcbio -inputs: -- id: qcout_rec - type: - items: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: File - name: qcout_rec - type: record - type: array -outputs: -- id: summary__multiqc - type: - items: - - File - - 'null' - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/pipeline_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/pipeline_summary.cwl deleted file mode 100644 index 230b84afc66..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/pipeline_summary.cwl +++ /dev/null @@ -1,155 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=qcout_rec:summary__qc;summary__metrics;resources;description;reference__fasta__base;config__algorithm__coverage_interval;genome_build;genome_resources__rnaseq__transcripts;config__algorithm__tools_off;config__algorithm__qc;analysis;config__algorithm__tools_on;align_bam -- sentinel_inputs=qc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- pipeline_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1024 - ramMin: 4096 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: fastqc=0.11.7=4 - specs: - - https://anaconda.org/bioconda/fastqc=0.11.7=4 - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: hts-nim-tools - specs: - - https://anaconda.org/bioconda/hts-nim-tools - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: qsignature - specs: - - https://anaconda.org/bioconda/qsignature - - package: qualimap - specs: - - https://anaconda.org/bioconda/qualimap - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: qc_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: File - name: qc_rec - type: record -outputs: -- id: qcout_rec - type: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: File - name: qcout_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/prepare_sample.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/prepare_sample.cwl deleted file mode 100644 index e5b8cdf645e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/prepare_sample.cwl +++ /dev/null @@ -1,146 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=prep_rec:resources;description;files;reference__fasta__base;config__algorithm__expression_caller;rgnames__lb;rgnames__rg;reference__hisat2__indexes;config__algorithm__fusion_caller;config__algorithm__aligner;rgnames__pl;genome_build;rgnames__pu;genome_resources__rnaseq__transcripts;config__algorithm__quality_format;analysis;rgnames__sample;rgnames__lane -- sentinel_inputs=files:var,rgnames__sample:var,reference__fasta__base:var,genome_build:var,genome_resources__rnaseq__transcripts:var,analysis:var,rgnames__pl:var,rgnames__pu:var,rgnames__lane:var,rgnames__rg:var,rgnames__lb:var,reference__hisat2__indexes:var,config__algorithm__aligner:var,config__algorithm__expression_caller:var,config__algorithm__fusion_caller:var,config__algorithm__quality_format:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prepare_sample -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1024 - ramMin: 4096 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 -inputs: -- id: files - type: - items: File - type: array -- id: rgnames__sample - type: string -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: File -- id: genome_build - type: string -- id: genome_resources__rnaseq__transcripts - secondaryFiles: - - .db - type: File -- id: analysis - type: string -- id: rgnames__pl - type: string -- id: rgnames__pu - type: string -- id: rgnames__lane - type: string -- id: rgnames__rg - type: string -- id: rgnames__lb - type: - - 'null' - - string -- id: reference__hisat2__indexes - type: File -- id: config__algorithm__aligner - type: string -- id: config__algorithm__expression_caller - type: - items: string - type: array -- id: config__algorithm__fusion_caller - type: - items: string - type: array -- id: config__algorithm__quality_format - type: string -- id: resources - type: string -- id: description - type: string -outputs: -- id: prep_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: files - type: - items: File - type: array - - name: reference__fasta__base - type: File - - name: config__algorithm__expression_caller - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: reference__hisat2__indexes - type: File - - name: config__algorithm__fusion_caller - type: - items: string - type: array - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: genome_build - type: string - - name: rgnames__pu - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__quality_format - type: string - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: rgnames__lane - type: string - name: prep_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/process_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/process_alignment.cwl deleted file mode 100644 index 88af1224f95..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/process_alignment.cwl +++ /dev/null @@ -1,108 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=align_bam -- sentinel_inputs=trim_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- process_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 4096 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: star - specs: - - https://anaconda.org/bioconda/star - - package: hisat2 - specs: - - https://anaconda.org/bioconda/hisat2 - - package: tophat - specs: - - https://anaconda.org/bioconda/tophat - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: seqtk - specs: - - https://anaconda.org/bioconda/seqtk -inputs: -- id: trim_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: files - type: - items: File - type: array - - name: reference__fasta__base - type: File - - name: config__algorithm__expression_caller - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: reference__hisat2__indexes - type: File - - name: config__algorithm__fusion_caller - type: - items: string - type: array - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: genome_build - type: string - - name: rgnames__pu - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__quality_format - type: string - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: rgnames__lane - type: string - name: trim_rec - type: record -outputs: -- id: align_bam - secondaryFiles: - - .bai - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/qc_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/qc_to_rec.cwl deleted file mode 100644 index 5ff5b2f10e0..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/qc_to_rec.cwl +++ /dev/null @@ -1,135 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=qc_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;genome_build;genome_resources__rnaseq__transcripts;config__algorithm__tools_off;config__algorithm__qc;analysis;config__algorithm__tools_on;align_bam -- sentinel_inputs=align_bam:var,analysis:var,reference__fasta__base:var,genome_resources__rnaseq__transcripts:var,genome_build:var,config__algorithm__coverage_interval:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,config__algorithm__qc:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- qc_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 2048 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: File - type: array -- id: analysis - type: - items: string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: genome_resources__rnaseq__transcripts - secondaryFiles: - - .db - type: - items: File - type: array -- id: genome_build - type: - items: string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__tools_on - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: qc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: File - name: qc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/rnaseq_quantitate.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/rnaseq_quantitate.cwl deleted file mode 100644 index 05d2ec813ae..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/rnaseq_quantitate.cwl +++ /dev/null @@ -1,123 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=count_file,quant__tsv,quant__hdf5,quant__fusion -- sentinel_inputs=trim_rec:record,align_bam:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- rnaseq_quantitate -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1025 - ramMin: 4096 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: sailfish - specs: - - https://anaconda.org/bioconda/sailfish - - package: salmon - specs: - - https://anaconda.org/bioconda/salmon - - package: kallisto> - specs: - - https://anaconda.org/bioconda/kallisto> - version: - - 0.43.1 - - package: subread - specs: - - https://anaconda.org/bioconda/subread - - package: gffread - specs: - - https://anaconda.org/bioconda/gffread - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-wasabi - specs: - - https://anaconda.org/bioconda/r-wasabi -inputs: -- id: trim_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: files - type: - items: File - type: array - - name: reference__fasta__base - type: File - - name: config__algorithm__expression_caller - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: reference__hisat2__indexes - type: File - - name: config__algorithm__fusion_caller - type: - items: string - type: array - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: genome_build - type: string - - name: rgnames__pu - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__quality_format - type: string - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: rgnames__lane - type: string - name: trim_rec - type: record -- id: align_bam - secondaryFiles: - - .bai - type: File -outputs: -- id: count_file - type: File -- id: quant__tsv - type: File -- id: quant__hdf5 - type: File -- id: quant__fusion - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/trim_sample.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/trim_sample.cwl deleted file mode 100644 index 14124636cec..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/rnaseq-workflow/steps/trim_sample.cwl +++ /dev/null @@ -1,140 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=trim_rec:resources;description;files;reference__fasta__base;config__algorithm__expression_caller;rgnames__lb;rgnames__rg;reference__hisat2__indexes;config__algorithm__fusion_caller;config__algorithm__aligner;rgnames__pl;genome_build;rgnames__pu;genome_resources__rnaseq__transcripts;config__algorithm__quality_format;analysis;rgnames__sample;rgnames__lane -- sentinel_inputs=prep_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- trim_sample -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-rnaseq:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1024 - ramMin: 4096 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: atropos;env - specs: - - https://anaconda.org/bioconda/atropos;env - version: - - python3 -inputs: -- id: prep_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: files - type: - items: File - type: array - - name: reference__fasta__base - type: File - - name: config__algorithm__expression_caller - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: reference__hisat2__indexes - type: File - - name: config__algorithm__fusion_caller - type: - items: string - type: array - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: genome_build - type: string - - name: rgnames__pu - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__quality_format - type: string - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: rgnames__lane - type: string - name: prep_rec - type: record -outputs: -- id: trim_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: files - type: - items: File - type: array - - name: reference__fasta__base - type: File - - name: config__algorithm__expression_caller - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: reference__hisat2__indexes - type: File - - name: config__algorithm__fusion_caller - type: - items: string - type: array - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: genome_build - type: string - - name: rgnames__pu - type: string - - name: genome_resources__rnaseq__transcripts - type: File - - name: config__algorithm__quality_format - type: string - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: rgnames__lane - type: string - name: trim_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/main-somatic-samples.json b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/main-somatic-samples.json deleted file mode 100644 index cbafda3641d..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/main-somatic-samples.json +++ /dev/null @@ -1,680 +0,0 @@ -{ - "analysis": [ - "variant2", - "variant2" - ], - "config__algorithm__adapters": [ - [], - [] - ], - "config__algorithm__align_split_size": [ - null, - null - ], - "config__algorithm__aligner": [ - "bwa", - "bwa" - ], - "config__algorithm__bam_clean": [ - "False", - "False" - ], - "config__algorithm__coverage_interval": [ - null, - null - ], - "config__algorithm__effects": [ - "snpeff", - "snpeff" - ], - "config__algorithm__ensemble": [ - null, - null - ], - "config__algorithm__exclude_regions": [ - [], - [] - ], - "config__algorithm__mark_duplicates": [ - "True", - "True" - ], - "config__algorithm__min_allele_fraction": [ - 10.0, - 10.0 - ], - "config__algorithm__nomap_split_size": [ - 250, - 250 - ], - "config__algorithm__nomap_split_targets": [ - 20, - 20 - ], - "config__algorithm__qc": [ - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "samtools", - "variants", - "viral" - ], - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "samtools", - "variants", - "viral" - ] - ], - "config__algorithm__recalibrate": [ - "False", - "False" - ], - "config__algorithm__svcaller": [ - [ - "manta" - ], - [ - "manta" - ] - ], - "config__algorithm__svprioritize": [ - null, - null - ], - "config__algorithm__svvalidate": [ - null, - null - ], - "config__algorithm__tools_off": [ - [], - [] - ], - "config__algorithm__tools_on": [ - [], - [] - ], - "config__algorithm__trim_reads": [ - "False", - "False" - ], - "config__algorithm__validate": [ - null, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/reference_material/Test1-grade.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/reference_material/Test1-grade.vcf.gz.tbi" - } - ] - } - ], - "config__algorithm__validate_regions": [ - null, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - } - ], - "config__algorithm__variant_regions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - } - ], - "config__algorithm__variantcaller": [ - [ - "vardict" - ], - [ - "vardict" - ] - ], - "config__algorithm__vcfanno": [ - [], - [] - ], - "description": [ - "Test2", - "Test1" - ], - "files": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/6_100326_FC6107FAAXX.bam" - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam.bai" - } - ] - } - ] - ], - "genome_build": [ - "hg19", - "hg19" - ], - "genome_resources__aliases__ensembl": [ - "homo_sapiens_vep_83_GRCh37", - "homo_sapiens_vep_83_GRCh37" - ], - "genome_resources__aliases__human": [ - "True", - "True" - ], - "genome_resources__aliases__snpeff": [ - "hg19", - "hg19" - ], - "genome_resources__rnaseq__gene_bed": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - } - ], - "genome_resources__variation__1000g": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__clinvar": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__cosmic": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__dbsnp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__encode_blacklist": [ - null, - null - ], - "genome_resources__variation__esp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__exac": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__gc_profile": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/gc/GC_profile.1000bp.cnp" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/gc/GC_profile.1000bp.cnp" - } - ], - "genome_resources__variation__germline_het_pon": [ - null, - null - ], - "genome_resources__variation__gnomad_exome": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__lcr": [ - null, - null - ], - "genome_resources__variation__polyx": [ - null, - null - ], - "genome_resources__variation__train_hapmap": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__train_indels": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - } - ], - "metadata__batch": [ - "b1", - "b1" - ], - "metadata__phenotype": [ - "normal", - "tumor" - ], - "reference__bwa__indexes": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.sa" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.sa" - } - ] - } - ], - "reference__fasta__base": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - } - ], - "reference__genome_context": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ] - ], - "reference__rtg": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - } - ], - "reference__snpeff__hg19": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - } - ], - "reference__versions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - } - ], - "reference__viral": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.amb" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.sa" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.amb" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.sa" - } - ] - } - ], - "resources": [ - "{}", - "{}" - ], - "rgnames__lane": [ - "Test2", - "Test1" - ], - "rgnames__lb": [ - null, - null - ], - "rgnames__pl": [ - "illumina", - "illumina" - ], - "rgnames__pu": [ - "Test2", - "Test1" - ], - "rgnames__rg": [ - "Test2", - "Test1" - ], - "rgnames__sample": [ - "Test2", - "Test1" - ], - "vrn_file": [ - null, - null - ] -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/main-somatic.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/main-somatic.cwl deleted file mode 100644 index 699d99a4690..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/main-somatic.cwl +++ /dev/null @@ -1,973 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: files - type: - items: - items: File - type: array - type: array -- id: config__algorithm__trim_reads - type: - items: - - string - - 'null' - - boolean - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: config__algorithm__vcfanno - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__svprioritize - type: - items: - - 'null' - - string - type: array -- id: resources - type: - items: string - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__adapters - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__gc_profile - type: - items: File - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__versions - type: - items: File - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__validate - secondaryFiles: - - .tbi - type: - items: - - 'null' - - File - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: description - type: - items: string - type: array -- id: config__algorithm__validate_regions - type: - items: - - 'null' - - File - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: genome_resources__variation__germline_het_pon - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: reference__viral - secondaryFiles: - - .amb - - .ann - - .sa - - .pac - - ^.dict - - .bwt - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__tools_on - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: config__algorithm__svvalidate - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__rtg - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -outputs: -- id: rgnames__sample_out - outputSource: prep_samples/rgnames__sample - type: - items: string - type: array -- id: align_bam - outputSource: postprocess_alignment/align_bam - type: - items: - - File - - 'null' - type: array -- id: regions__sample_callable - outputSource: postprocess_alignment/regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: validate__grading_summary - outputSource: summarize_vc/validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: variants__calls - outputSource: summarize_vc/variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - outputSource: summarize_vc/variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: sv__calls - outputSource: summarize_sv/sv__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: svvalidate__grading_summary - outputSource: summarize_sv/svvalidate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: sv__prioritize__tsv - outputSource: summarize_sv/sv__prioritize__tsv - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__prioritize__raw - outputSource: summarize_sv/sv__prioritize__raw - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__supplemental - outputSource: summarize_sv/sv__supplemental - type: - items: - items: - - File - type: array - type: array -- id: summary__multiqc - outputSource: multiqc_summary/summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - outputSource: multiqc_summary/versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - outputSource: multiqc_summary/versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: alignment_to_rec - in: - - id: files - source: files - - id: analysis - source: analysis - - id: config__algorithm__align_split_size - source: config__algorithm__align_split_size - - id: reference__fasta__base - source: reference__fasta__base - - id: rgnames__pl - source: rgnames__pl - - id: rgnames__sample - source: rgnames__sample - - id: rgnames__pu - source: rgnames__pu - - id: rgnames__lane - source: rgnames__lane - - id: rgnames__rg - source: rgnames__rg - - id: rgnames__lb - source: rgnames__lb - - id: reference__bwa__indexes - source: reference__bwa__indexes - - id: config__algorithm__aligner - source: config__algorithm__aligner - - id: config__algorithm__trim_reads - source: config__algorithm__trim_reads - - id: config__algorithm__adapters - source: config__algorithm__adapters - - id: config__algorithm__bam_clean - source: config__algorithm__bam_clean - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: config__algorithm__mark_duplicates - source: config__algorithm__mark_duplicates - - id: resources - source: resources - - id: description - source: description - out: - - id: alignment_rec - run: steps/alignment_to_rec.cwl -- id: alignment - in: - - id: alignment_rec - source: alignment_to_rec/alignment_rec - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: wf-alignment.cwl - scatter: - - alignment_rec - scatterMethod: dotproduct -- id: prep_samples_to_rec - in: - - id: rgnames__sample - source: rgnames__sample - - id: config__algorithm__svcaller - source: config__algorithm__svcaller - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: prep_samples_rec - run: steps/prep_samples_to_rec.cwl -- id: prep_samples - in: - - id: prep_samples_rec - source: prep_samples_to_rec/prep_samples_rec - out: - - id: rgnames__sample - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - run: steps/prep_samples.cwl - scatter: - - prep_samples_rec - scatterMethod: dotproduct -- id: postprocess_alignment_to_rec - in: - - id: align_bam - source: alignment/align_bam - - id: config__algorithm__coverage_interval - source: config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: prep_samples/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: prep_samples/config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - source: prep_samples/config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - source: prep_samples/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: prep_samples/config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - source: prep_samples/config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - source: prep_samples/config__algorithm__seq2c_bed_ready - - id: config__algorithm__recalibrate - source: config__algorithm__recalibrate - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: postprocess_alignment_rec - run: steps/postprocess_alignment_to_rec.cwl -- id: postprocess_alignment - in: - - id: postprocess_alignment_rec - source: postprocess_alignment_to_rec/postprocess_alignment_rec - out: - - id: config__algorithm__coverage_interval - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - - id: regions__callable - - id: regions__sample_callable - - id: regions__nblock - - id: depth__samtools__stats - - id: depth__samtools__idxstats - - id: depth__variant_regions__regions - - id: depth__variant_regions__dist - - id: depth__sv_regions__regions - - id: depth__sv_regions__dist - - id: depth__coverage__regions - - id: depth__coverage__dist - - id: depth__coverage__thresholds - - id: align_bam - run: steps/postprocess_alignment.cwl - scatter: - - postprocess_alignment_rec - scatterMethod: dotproduct -- id: combine_sample_regions - in: - - id: regions__callable - source: postprocess_alignment/regions__callable - - id: regions__nblock - source: postprocess_alignment/regions__nblock - - id: metadata__batch - source: metadata__batch - - id: config__algorithm__nomap_split_size - source: config__algorithm__nomap_split_size - - id: config__algorithm__nomap_split_targets - source: config__algorithm__nomap_split_targets - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: config__algorithm__callable_regions - - id: config__algorithm__non_callable_regions - - id: config__algorithm__callable_count - run: steps/combine_sample_regions.cwl -- id: batch_for_variantcall - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: align_bam - source: postprocess_alignment/align_bam - - id: vrn_file - source: vrn_file - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: config__algorithm__variantcaller - source: config__algorithm__variantcaller - - id: config__algorithm__ensemble - source: config__algorithm__ensemble - - id: config__algorithm__vcfanno - source: config__algorithm__vcfanno - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__effects - source: config__algorithm__effects - - id: config__algorithm__min_allele_fraction - source: config__algorithm__min_allele_fraction - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__validate - source: config__algorithm__validate - - id: config__algorithm__validate_regions - source: config__algorithm__validate_regions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__rtg - source: reference__rtg - - id: reference__genome_context - source: reference__genome_context - - id: genome_resources__variation__clinvar - source: genome_resources__variation__clinvar - - id: genome_resources__variation__cosmic - source: genome_resources__variation__cosmic - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__esp - source: genome_resources__variation__esp - - id: genome_resources__variation__exac - source: genome_resources__variation__exac - - id: genome_resources__variation__gnomad_exome - source: genome_resources__variation__gnomad_exome - - id: genome_resources__variation__1000g - source: genome_resources__variation__1000g - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__aliases__ensembl - source: genome_resources__aliases__ensembl - - id: genome_resources__aliases__human - source: genome_resources__aliases__human - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__hg19 - source: reference__snpeff__hg19 - - id: genome_resources__variation__train_hapmap - source: genome_resources__variation__train_hapmap - - id: genome_resources__variation__train_indels - source: genome_resources__variation__train_indels - - id: resources - source: resources - - id: description - source: description - out: - - id: batch_rec - run: steps/batch_for_variantcall.cwl -- id: variantcall - in: - - id: batch_rec - source: batch_for_variantcall/batch_rec - out: - - id: vc_rec - run: wf-variantcall.cwl - scatter: - - batch_rec - scatterMethod: dotproduct -- id: summarize_vc - in: - - id: vc_rec - source: variantcall/vc_rec - out: - - id: variants__calls - - id: variants__gvcf - - id: variants__samples - - id: validate__grading_summary - - id: validate__grading_plots - run: steps/summarize_vc.cwl -- id: calculate_sv_bins - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: reference__fasta__base - source: reference__fasta__base - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__seq2c_bed_ready - source: postprocess_alignment/config__algorithm__seq2c_bed_ready - - id: config__algorithm__svcaller - source: config__algorithm__svcaller - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: resources - source: resources - - id: description - source: description - out: - - id: sv_bin_rec - run: steps/calculate_sv_bins.cwl -- id: calculate_sv_coverage - in: - - id: sv_bin_rec - source: calculate_sv_bins/sv_bin_rec - out: - - id: sv_rawcoverage_rec - run: steps/calculate_sv_coverage.cwl - scatter: - - sv_bin_rec - scatterMethod: dotproduct -- id: normalize_sv_coverage - in: - - id: sv_rawcoverage_rec - source: calculate_sv_coverage/sv_rawcoverage_rec - out: - - id: sv_coverage_rec - run: steps/normalize_sv_coverage.cwl -- id: batch_for_sv - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: work_bam_plus__disc - source: alignment/work_bam_plus__disc - - id: work_bam_plus__sr - source: alignment/work_bam_plus__sr - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: config__algorithm__svprioritize - source: config__algorithm__svprioritize - - id: config__algorithm__svvalidate - source: config__algorithm__svvalidate - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: genome_resources__variation__gc_profile - source: genome_resources__variation__gc_profile - - id: genome_resources__variation__germline_het_pon - source: genome_resources__variation__germline_het_pon - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__hg19 - source: reference__snpeff__hg19 - - id: sv_coverage_rec - source: normalize_sv_coverage/sv_coverage_rec - - id: variants__samples - source: summarize_vc/variants__samples - out: - - id: sv_batch_rec - run: steps/batch_for_sv.cwl -- id: svcall - in: - - id: sv_batch_rec - source: batch_for_sv/sv_batch_rec - out: - - id: sv_rec - run: wf-svcall.cwl - scatter: - - sv_batch_rec - scatterMethod: dotproduct -- id: summarize_sv - in: - - id: sv_rec - source: svcall/sv_rec - out: - - id: sv__calls - - id: sv__supplemental - - id: sv__prioritize__tsv - - id: sv__prioritize__raw - - id: svvalidate__grading_summary - - id: svvalidate__grading_plots - run: steps/summarize_sv.cwl -- id: qc_to_rec - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: analysis - source: analysis - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__versions - source: reference__versions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: genome_build - source: genome_build - - id: config__algorithm__qc - source: config__algorithm__qc - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: depth__variant_regions__dist - source: postprocess_alignment/depth__variant_regions__dist - - id: depth__samtools__stats - source: postprocess_alignment/depth__samtools__stats - - id: depth__samtools__idxstats - source: postprocess_alignment/depth__samtools__idxstats - - id: depth__sv_regions__regions - source: postprocess_alignment/depth__sv_regions__regions - - id: depth__sv_regions__dist - source: postprocess_alignment/depth__sv_regions__dist - - id: depth__coverage__regions - source: postprocess_alignment/depth__coverage__regions - - id: depth__coverage__dist - source: postprocess_alignment/depth__coverage__dist - - id: depth__coverage__thresholds - source: postprocess_alignment/depth__coverage__thresholds - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__coverage - source: postprocess_alignment/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: postprocess_alignment/config__algorithm__coverage_merged - - id: variants__samples - source: summarize_vc/variants__samples - - id: reference__viral - source: reference__viral - - id: resources - source: resources - - id: description - source: description - out: - - id: qc_rec - run: steps/qc_to_rec.cwl -- id: pipeline_summary - in: - - id: qc_rec - source: qc_to_rec/qc_rec - out: - - id: qcout_rec - run: steps/pipeline_summary.cwl - scatter: - - qc_rec - scatterMethod: dotproduct -- id: multiqc_summary - in: - - id: qcout_rec - source: pipeline_summary/qcout_rec - out: - - id: summary__multiqc - - id: versions__tools - - id: versions__data - run: steps/multiqc_summary.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/alignment_to_rec.cwl deleted file mode 100644 index 11cd9fc8d40..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/alignment_to_rec.cwl +++ /dev/null @@ -1,200 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=alignment_rec:resources;description;config__algorithm__align_split_size;files;config__algorithm__trim_reads;reference__fasta__base;config__algorithm__adapters;rgnames__lb;rgnames__rg;rgnames__lane;reference__bwa__indexes;config__algorithm__bam_clean;config__algorithm__aligner;rgnames__pl;rgnames__pu;config__algorithm__mark_duplicates;analysis;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=files:var,analysis:var,config__algorithm__align_split_size:var,reference__fasta__base:var,rgnames__pl:var,rgnames__sample:var,rgnames__pu:var,rgnames__lane:var,rgnames__rg:var,rgnames__lb:var,reference__bwa__indexes:var,config__algorithm__aligner:var,config__algorithm__trim_reads:var,config__algorithm__adapters:var,config__algorithm__bam_clean:var,config__algorithm__variant_regions:var,config__algorithm__mark_duplicates:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10244 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: files - type: - items: - items: File - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: config__algorithm__trim_reads - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__adapters - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/batch_for_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/batch_for_sv.cwl deleted file mode 100644 index 16b905acacf..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/batch_for_sv.cwl +++ /dev/null @@ -1,375 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=sv_batch_rec:resources;description;config__algorithm__svprioritize;genome_resources__variation__gc_profile;reference__snpeff__hg19;genome_build;genome_resources__variation__germline_het_pon;config__algorithm__tools_off;analysis;config__algorithm__tools_on;config__algorithm__svvalidate;genome_resources__aliases__snpeff;work_bam_plus__disc;work_bam_plus__sr;regions__sample_callable;variants__samples;depth__bins__normalized;depth__bins__background;depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,work_bam_plus__disc:var,work_bam_plus__sr:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,config__algorithm__svprioritize:var,config__algorithm__svvalidate:var,regions__sample_callable:var,genome_resources__variation__gc_profile:var,genome_resources__variation__germline_het_pon:var,genome_resources__aliases__snpeff:var,reference__snpeff__hg19:var,sv_coverage_rec:record,variants__samples:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10240 - ramMin: 6144 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__tools_on - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__svprioritize - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__svvalidate - type: - items: - - 'null' - - string - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: genome_resources__variation__gc_profile - type: - items: File - type: array -- id: genome_resources__variation__germline_het_pon - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: sv_coverage_rec - type: - items: - fields: - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_coverage_rec - type: record - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -outputs: -- id: sv_batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: File - - name: reference__snpeff__hg19 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/batch_for_variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/batch_for_variantcall.cwl deleted file mode 100644 index 60af6c4469e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/batch_for_variantcall.cwl +++ /dev/null @@ -1,407 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=batch_rec:resources;description;reference__fasta__base;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;genome_resources__variation__1000g;config__algorithm__min_allele_fraction;vrn_file;genome_resources__variation__train_hapmap;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;metadata__phenotype;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;align_bam;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,align_bam:var,vrn_file:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,regions__sample_callable:var,config__algorithm__variantcaller:var,config__algorithm__ensemble:var,config__algorithm__vcfanno:var,config__algorithm__coverage_interval:var,config__algorithm__effects:var,config__algorithm__min_allele_fraction:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__validate:var,config__algorithm__validate_regions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,reference__fasta__base:var,reference__rtg:var,reference__genome_context:var,genome_resources__variation__clinvar:var,genome_resources__variation__cosmic:var,genome_resources__variation__dbsnp:var,genome_resources__variation__esp:var,genome_resources__variation__exac:var,genome_resources__variation__gnomad_exome:var,genome_resources__variation__1000g:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__aliases__ensembl:var,genome_resources__aliases__human:var,genome_resources__aliases__snpeff:var,reference__snpeff__hg19:var,genome_resources__variation__train_hapmap:var,genome_resources__variation__train_indels:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_variantcall -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10246 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__vcfanno - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__validate - secondaryFiles: - - .tbi - type: - items: - - 'null' - - File - type: array -- id: config__algorithm__validate_regions - type: - items: - - 'null' - - File - type: array -- id: config__algorithm__tools_on - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__rtg - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/calculate_sv_bins.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/calculate_sv_bins.cwl deleted file mode 100644 index 40bd035478a..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/calculate_sv_bins.cwl +++ /dev/null @@ -1,225 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv_bin_rec:regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;resources;description;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=align_bam:var,reference__fasta__base:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__seq2c_bed_ready:var,config__algorithm__svcaller:var,depth__variant_regions__regions:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__rnaseq__gene_bed:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- calculate_sv_bins -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10244 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__seq2c_bed_ready - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: sv_bin_rec - type: - items: - fields: - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_bin_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/calculate_sv_coverage.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/calculate_sv_coverage.cwl deleted file mode 100644 index fc75d2c1b71..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/calculate_sv_coverage.cwl +++ /dev/null @@ -1,218 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=sv_rawcoverage_rec:depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;resources;description;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_bin_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- calculate_sv_coverage -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10244 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit - - package: seq2c - specs: - - https://anaconda.org/bioconda/seq2c -inputs: -- id: sv_bin_rec - type: - fields: - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_bin_rec - type: record -outputs: -- id: sv_rawcoverage_rec - type: - fields: - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rawcoverage_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/combine_sample_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/combine_sample_regions.cwl deleted file mode 100644 index 5a8f8a3c6cb..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/combine_sample_regions.cwl +++ /dev/null @@ -1,99 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=config__algorithm__callable_regions,config__algorithm__non_callable_regions,config__algorithm__callable_count -- sentinel_inputs=regions__callable:var,regions__nblock:var,metadata__batch:var,config__algorithm__nomap_split_size:var,config__algorithm__nomap_split_targets:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- combine_sample_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10241 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: regions__callable - type: - items: - - File - - 'null' - type: array -- id: regions__nblock - type: - items: - - File - - 'null' - type: array -- id: metadata__batch - type: - items: string - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__non_callable_regions - type: - items: File - type: array -- id: config__algorithm__callable_count - type: - items: int - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/compare_to_rm.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/compare_to_rm.cwl deleted file mode 100644 index d6b910b7f9e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/compare_to_rm.cwl +++ /dev/null @@ -1,311 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vc_rec:batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;resources;description;vrn_file;reference__fasta__base;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;metadata__batch;config__algorithm__min_allele_fraction;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;metadata__phenotype;genome_resources__aliases__human;config__algorithm__tools_off;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- compare_to_rm -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10244 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: gvcf-regions - specs: - - https://anaconda.org/bioconda/gvcf-regions - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: rtg-tools - specs: - - https://anaconda.org/bioconda/rtg-tools - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file - secondaryFiles: - - .tbi - type: File -outputs: -- id: vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/concat_batch_variantcalls.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/concat_batch_variantcalls.cwl deleted file mode 100644 index ef31b2ffe4f..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/concat_batch_variantcalls.cwl +++ /dev/null @@ -1,198 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-merge -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,region_block:var,vrn_file_region:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- concat_batch_variantcalls -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10244 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block - type: - items: - items: string - type: array - type: array -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/detect_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/detect_sv.cwl deleted file mode 100644 index 234bcc60d69..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/detect_sv.cwl +++ /dev/null @@ -1,414 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=sv_rec:sv__variantcaller;sv__vrn_file;sv__supplemental;svvalidate__summary;resources;description;config__algorithm__svprioritize;genome_resources__variation__gc_profile;genome_build;genome_resources__variation__germline_het_pon;config__algorithm__tools_off;analysis;config__algorithm__tools_on;config__algorithm__svvalidate;genome_resources__aliases__snpeff;regions__sample_callable;variants__samples;depth__bins__normalized;depth__bins__background;depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- detect_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10246 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit - - package: delly - specs: - - https://anaconda.org/bioconda/delly - - package: duphold - specs: - - https://anaconda.org/bioconda/duphold - - package: extract-sv-reads - specs: - - https://anaconda.org/bioconda/extract-sv-reads - - package: lumpy-sv - specs: - - https://anaconda.org/bioconda/lumpy-sv - - package: manta - specs: - - https://anaconda.org/bioconda/manta - - package: break-point-inspector - specs: - - https://anaconda.org/bioconda/break-point-inspector - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: smoove - specs: - - https://anaconda.org/bioconda/smoove - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: seq2c - specs: - - https://anaconda.org/bioconda/seq2c - - package: simple_sv_annotation - specs: - - https://anaconda.org/bioconda/simple_sv_annotation - - package: survivor - specs: - - https://anaconda.org/bioconda/survivor - - package: svtools - specs: - - https://anaconda.org/bioconda/svtools - - package: svtyper - specs: - - https://anaconda.org/bioconda/svtyper - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: xorg-libxt - specs: - - https://anaconda.org/bioconda/xorg-libxt - - package: vawk - specs: - - https://anaconda.org/bioconda/vawk -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: sv_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: File - - name: reference__snpeff__hg19 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array -outputs: -- id: sv_rec - type: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/get_parallel_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/get_parallel_regions.cwl deleted file mode 100644 index 48e5f050eb3..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/get_parallel_regions.cwl +++ /dev/null @@ -1,173 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-split -- sentinel_outputs=region_block -- sentinel_inputs=batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- get_parallel_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10246 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: region_block - type: - items: - items: string - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/merge_split_alignments.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/merge_split_alignments.cwl deleted file mode 100644 index 029b704bbb2..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/merge_split_alignments.cwl +++ /dev/null @@ -1,169 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-merge -- sentinel_outputs=align_bam,work_bam_plus__disc,work_bam_plus__sr,hla__fastq -- sentinel_inputs=alignment_rec:record,work_bam:var,align_bam:var,work_bam_plus__disc:var,work_bam_plus__sr:var,hla__fastq:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- merge_split_alignments -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10251 - ramMin: 6144 - tmpdirMin: 6 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -- id: work_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: align_bam_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__disc_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__sr_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: hla__fastq_toolinput - type: - items: - - 'null' - - items: File - type: array - type: array -outputs: -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/multiqc_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/multiqc_summary.cwl deleted file mode 100644 index 9d65504ba6b..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/multiqc_summary.cwl +++ /dev/null @@ -1,96 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=summary__multiqc,versions__tools,versions__data -- sentinel_inputs=qcout_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- multiqc_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10246 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: multiqc - specs: - - https://anaconda.org/bioconda/multiqc - - package: multiqc-bcbio - specs: - - https://anaconda.org/bioconda/multiqc-bcbio -inputs: -- id: qcout_rec - type: - items: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - name: qcout_rec - type: record - type: array -outputs: -- id: summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/normalize_sv_coverage.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/normalize_sv_coverage.cwl deleted file mode 100644 index 1bdc8dbc3b8..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/normalize_sv_coverage.cwl +++ /dev/null @@ -1,236 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv_coverage_rec:depth__bins__normalized;depth__bins__background;resources;description;depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_rawcoverage_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- normalize_sv_coverage -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10244 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit -inputs: -- id: sv_rawcoverage_rec - type: - items: - fields: - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rawcoverage_rec - type: record - type: array -outputs: -- id: sv_coverage_rec - type: - items: - fields: - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_coverage_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/pipeline_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/pipeline_summary.cwl deleted file mode 100644 index 89f6372949c..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/pipeline_summary.cwl +++ /dev/null @@ -1,223 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=qcout_rec:summary__qc;summary__metrics;description;reference__versions;genome_build;config__algorithm__tools_off;config__algorithm__qc;config__algorithm__tools_on -- sentinel_inputs=qc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- pipeline_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10246 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: fastqc=0.11.7=5 - specs: - - https://anaconda.org/bioconda/fastqc=0.11.7=5 - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: hts-nim-tools - specs: - - https://anaconda.org/bioconda/hts-nim-tools - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: qsignature - specs: - - https://anaconda.org/bioconda/qsignature - - package: qualimap - specs: - - https://anaconda.org/bioconda/qualimap - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: preseq - specs: - - https://anaconda.org/bioconda/preseq - - package: peddy - specs: - - https://anaconda.org/bioconda/peddy - - package: verifybamid2 - specs: - - https://anaconda.org/bioconda/verifybamid2 -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: qc_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: reference__viral - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record -outputs: -- id: qcout_rec - type: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - name: qcout_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_alignment.cwl deleted file mode 100644 index 8585bfb3074..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_alignment.cwl +++ /dev/null @@ -1,224 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=config__algorithm__coverage_interval,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready,regions__callable,regions__sample_callable,regions__nblock,depth__samtools__stats,depth__samtools__idxstats,depth__variant_regions__regions,depth__variant_regions__dist,depth__sv_regions__regions,depth__sv_regions__dist,depth__coverage__regions,depth__coverage__dist,depth__coverage__thresholds,align_bam -- sentinel_inputs=postprocess_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10249 - ramMin: 6144 - tmpdirMin: 5 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon -- class: arv:APIRequirement -inputs: -- id: postprocess_alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record -outputs: -- id: config__algorithm__coverage_interval - type: - - string - - 'null' -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -- id: regions__callable - type: - - File - - 'null' -- id: regions__sample_callable - type: - - File - - 'null' -- id: regions__nblock - type: - - File - - 'null' -- id: depth__samtools__stats - type: - - File - - 'null' -- id: depth__samtools__idxstats - type: - - File - - 'null' -- id: depth__variant_regions__regions - type: - - File - - 'null' -- id: depth__variant_regions__dist - type: - - File - - 'null' -- id: depth__sv_regions__regions - type: - - File - - 'null' -- id: depth__sv_regions__dist - type: - - File - - 'null' -- id: depth__coverage__regions - type: - - File - - 'null' -- id: depth__coverage__dist - type: - - File - - 'null' -- id: depth__coverage__thresholds - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_alignment_to_rec.cwl deleted file mode 100644 index 9d93ac8f1db..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_alignment_to_rec.cwl +++ /dev/null @@ -1,235 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=postprocess_alignment_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;genome_resources__variation__lcr;config__algorithm__recalibrate;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__tools_on;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__variant_regions_orig;config__algorithm__coverage;config__algorithm__coverage_merged;config__algorithm__coverage_orig;config__algorithm__seq2c_bed_ready -- sentinel_inputs=align_bam:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__variant_regions_orig:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,config__algorithm__coverage_orig:var,config__algorithm__seq2c_bed_ready:var,config__algorithm__recalibrate:var,config__algorithm__tools_on:var,genome_resources__rnaseq__gene_bed:var,genome_resources__variation__dbsnp:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10244 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__seq2c_bed_ready - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_on - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: postprocess_alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_variants.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_variants.cwl deleted file mode 100644 index 6fba055bcd7..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/postprocess_variants.cwl +++ /dev/null @@ -1,182 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_variants -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10241 - ramMin: 6144 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: snpeff - specs: - - https://anaconda.org/bioconda/snpeff - version: - - 4.3.1t -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file_toolinput - secondaryFiles: - - .tbi - type: File -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_align_inputs.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_align_inputs.cwl deleted file mode 100644 index c40a38fc01a..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_align_inputs.cwl +++ /dev/null @@ -1,143 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-split -- sentinel_outputs=process_alignment_rec:files;config__algorithm__quality_format;align_split -- sentinel_inputs=alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_align_inputs -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10244 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: atropos;env - specs: - - https://anaconda.org/bioconda/atropos;env - version: - - python3 - - package: optitype - specs: - - https://anaconda.org/bioconda/optitype - - package: razers3 - specs: - - https://anaconda.org/bioconda/razers3 - version: - - 3.5.0 - - package: coincbc - specs: - - https://anaconda.org/bioconda/coincbc -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -outputs: -- id: process_alignment_rec - type: - items: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_samples.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_samples.cwl deleted file mode 100644 index 4e62820988d..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_samples.cwl +++ /dev/null @@ -1,95 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=rgnames__sample,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready -- sentinel_inputs=prep_samples_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10241 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy -inputs: -- id: prep_samples_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record -outputs: -- id: rgnames__sample - type: string -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_samples_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_samples_to_rec.cwl deleted file mode 100644 index 9268efdafcb..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/prep_samples_to_rec.cwl +++ /dev/null @@ -1,85 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=prep_samples_rec:resources;description;reference__fasta__base;config__algorithm__svcaller;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=rgnames__sample:var,config__algorithm__svcaller:var,config__algorithm__variant_regions:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10241 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: prep_samples_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/process_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/process_alignment.cwl deleted file mode 100644 index 45b58d1ab98..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/process_alignment.cwl +++ /dev/null @@ -1,199 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-parallel -- sentinel_outputs=work_bam,align_bam,hla__fastq,work_bam_plus__disc,work_bam_plus__sr -- sentinel_inputs=alignment_rec:record,process_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- process_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10246 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 7 -- class: SoftwareRequirement - packages: - - package: bwa - specs: - - https://anaconda.org/bioconda/bwa - - package: bwakit - specs: - - https://anaconda.org/bioconda/bwakit - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: minimap2 - specs: - - https://anaconda.org/bioconda/minimap2 - - package: novoalign - specs: - - https://anaconda.org/bioconda/novoalign - - package: snap-aligner - specs: - - https://anaconda.org/bioconda/snap-aligner - version: - - 1.0dev.97 - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: fgbio - specs: - - https://anaconda.org/bioconda/fgbio - - package: umis - specs: - - https://anaconda.org/bioconda/umis - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: seqtk - specs: - - https://anaconda.org/bioconda/seqtk - - package: samblaster - specs: - - https://anaconda.org/bioconda/samblaster - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -- class: arv:APIRequirement -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -- id: process_alignment_rec - type: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record -outputs: -- id: work_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/qc_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/qc_to_rec.cwl deleted file mode 100644 index eccc8330618..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/qc_to_rec.cwl +++ /dev/null @@ -1,310 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=qc_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;metadata__batch;reference__versions;genome_build;metadata__phenotype;config__algorithm__tools_off;reference__viral;config__algorithm__qc;analysis;config__algorithm__tools_on;config__algorithm__variant_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__coverage;config__algorithm__coverage_merged;depth__samtools__stats;depth__samtools__idxstats;depth__variant_regions__regions;depth__variant_regions__dist;depth__sv_regions__regions;depth__sv_regions__dist;depth__coverage__regions;depth__coverage__dist;depth__coverage__thresholds;variants__samples -- sentinel_inputs=align_bam:var,analysis:var,reference__fasta__base:var,reference__versions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,genome_build:var,config__algorithm__qc:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__coverage_interval:var,depth__variant_regions__regions:var,depth__variant_regions__dist:var,depth__samtools__stats:var,depth__samtools__idxstats:var,depth__sv_regions__regions:var,depth__sv_regions__dist:var,depth__coverage__regions:var,depth__coverage__dist:var,depth__coverage__thresholds:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,variants__samples:var,reference__viral:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- qc_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10244 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: analysis - type: - items: string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__versions - type: - items: File - type: array -- id: config__algorithm__tools_on - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_build - type: - items: string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__variant_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__stats - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__idxstats - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__regions - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__thresholds - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: reference__viral - secondaryFiles: - - .amb - - .ann - - .sa - - .pac - - ^.dict - - .bwt - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: qc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: reference__viral - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/summarize_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/summarize_sv.cwl deleted file mode 100644 index be7b4a8c87f..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/summarize_sv.cwl +++ /dev/null @@ -1,242 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv__calls,sv__supplemental,sv__prioritize__tsv,sv__prioritize__raw,svvalidate__grading_summary,svvalidate__grading_plots -- sentinel_inputs=sv_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10243 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcbio-prioritize - specs: - - https://anaconda.org/bioconda/bcbio-prioritize -inputs: -- id: sv_rec - type: - items: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array - type: array -outputs: -- id: sv__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__supplemental - type: - items: - items: - - File - type: array - type: array -- id: sv__prioritize__tsv - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__prioritize__raw - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: svvalidate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: svvalidate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/summarize_vc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/summarize_vc.cwl deleted file mode 100644 index 62c64099b31..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/summarize_vc.cwl +++ /dev/null @@ -1,195 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=variants__calls,variants__gvcf,variants__samples,validate__grading_summary,validate__grading_plots -- sentinel_inputs=vc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_vc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10241 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: vc_rec - type: - items: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array - type: array -outputs: -- id: variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: validate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/variantcall_batch_region.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/variantcall_batch_region.cwl deleted file mode 100644 index 1dc7a6e7128..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/steps/variantcall_batch_region.cwl +++ /dev/null @@ -1,268 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-parallel -- sentinel_outputs=vrn_file_region,region_block -- sentinel_inputs=batch_rec:record,region_block:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- variantcall_batch_region -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 10246 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: freebayes - specs: - - https://anaconda.org/bioconda/freebayes - version: - - 1.1.0.46 - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: vqsr_cnn - specs: - - https://anaconda.org/bioconda/vqsr_cnn - - package: deepvariant - specs: - - https://anaconda.org/bioconda/deepvariant - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: octopus - specs: - - https://anaconda.org/bioconda/octopus - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: platypus-variant - specs: - - https://anaconda.org/bioconda/platypus-variant - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: strelka - specs: - - https://anaconda.org/bioconda/strelka - - package: vardict - specs: - - https://anaconda.org/bioconda/vardict - - package: vardict-java - specs: - - https://anaconda.org/bioconda/vardict-java - - package: varscan - specs: - - https://anaconda.org/bioconda/varscan - - package: moreutils - specs: - - https://anaconda.org/bioconda/moreutils - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno - - package: vcflib - specs: - - https://anaconda.org/bioconda/vcflib - - package: vt - specs: - - https://anaconda.org/bioconda/vt - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: perl - specs: - - https://anaconda.org/bioconda/perl -- class: arv:APIRequirement -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block_toolinput - type: - items: string - type: array -outputs: -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - - File - - 'null' -- id: region_block - type: - items: string - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-alignment.cwl deleted file mode 100644 index c2ce73cfc61..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-alignment.cwl +++ /dev/null @@ -1,144 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: alignment_rec - type: record -outputs: -- id: align_bam - outputSource: merge_split_alignments/align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - outputSource: merge_split_alignments/work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - outputSource: merge_split_alignments/work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - outputSource: merge_split_alignments/hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: prep_align_inputs - in: - - id: alignment_rec - source: alignment_rec - out: - - id: process_alignment_rec - run: steps/prep_align_inputs.cwl -- id: process_alignment - in: - - id: alignment_rec - source: alignment_rec - - id: process_alignment_rec - source: prep_align_inputs/process_alignment_rec - out: - - id: work_bam - - id: align_bam - - id: hla__fastq - - id: work_bam_plus__disc - - id: work_bam_plus__sr - run: steps/process_alignment.cwl - scatter: - - process_alignment_rec - scatterMethod: dotproduct -- id: merge_split_alignments - in: - - id: alignment_rec - source: alignment_rec - - id: work_bam - source: process_alignment/work_bam - - id: align_bam_toolinput - source: process_alignment/align_bam - - id: work_bam_plus__disc_toolinput - source: process_alignment/work_bam_plus__disc - - id: work_bam_plus__sr_toolinput - source: process_alignment/work_bam_plus__sr - - id: hla__fastq_toolinput - source: process_alignment/hla__fastq - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: steps/merge_split_alignments.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-svcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-svcall.cwl deleted file mode 100644 index db9522935a1..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-svcall.cwl +++ /dev/null @@ -1,328 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: sv_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: File - - name: reference__snpeff__hg19 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array -outputs: -- id: sv_rec - outputSource: detect_sv/sv_rec - type: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: detect_sv - in: - - id: sv_batch_rec - source: sv_batch_rec - out: - - id: sv_rec - run: steps/detect_sv.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-variantcall.cwl deleted file mode 100644 index da6bab7ac21..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/somatic-workflow/wf-variantcall.cwl +++ /dev/null @@ -1,312 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: vc_rec - outputSource: compare_to_rm/vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - File - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - File - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: get_parallel_regions - in: - - id: batch_rec - source: batch_rec - out: - - id: region_block - run: steps/get_parallel_regions.cwl -- id: variantcall_batch_region - in: - - id: batch_rec - source: batch_rec - - id: region_block_toolinput - source: get_parallel_regions/region_block - out: - - id: vrn_file_region - - id: region_block - run: steps/variantcall_batch_region.cwl - scatter: - - region_block_toolinput - scatterMethod: dotproduct -- id: concat_batch_variantcalls - in: - - id: batch_rec - source: batch_rec - - id: region_block - source: variantcall_batch_region/region_block - - id: vrn_file_region - source: variantcall_batch_region/vrn_file_region - out: - - id: vrn_file - run: steps/concat_batch_variantcalls.cwl -- id: postprocess_variants - in: - - id: batch_rec - source: batch_rec - - id: vrn_file_toolinput - source: concat_batch_variantcalls/vrn_file - out: - - id: vrn_file - run: steps/postprocess_variants.cwl -- id: compare_to_rm - in: - - id: batch_rec - source: batch_rec - - id: vrn_file - source: postprocess_variants/vrn_file - out: - - id: vc_rec - run: steps/compare_to_rm.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/main-svcall-samples.json b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/main-svcall-samples.json deleted file mode 100644 index 28826670c5c..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/main-svcall-samples.json +++ /dev/null @@ -1,707 +0,0 @@ -{ - "analysis": [ - "variant", - "variant" - ], - "config__algorithm__adapters": [ - [], - [] - ], - "config__algorithm__align_split_size": [ - null, - null - ], - "config__algorithm__aligner": [ - "bwa", - "bwa" - ], - "config__algorithm__bam_clean": [ - "False", - "False" - ], - "config__algorithm__coverage_interval": [ - null, - null - ], - "config__algorithm__effects": [ - "snpeff", - "snpeff" - ], - "config__algorithm__ensemble": [ - null, - null - ], - "config__algorithm__exclude_regions": [ - [], - [] - ], - "config__algorithm__mark_duplicates": [ - "True", - "True" - ], - "config__algorithm__min_allele_fraction": [ - 10.0, - 10.0 - ], - "config__algorithm__nomap_split_size": [ - 250, - 250 - ], - "config__algorithm__nomap_split_targets": [ - 20, - 20 - ], - "config__algorithm__qc": [ - [ - "samtools" - ], - [ - "samtools" - ] - ], - "config__algorithm__recalibrate": [ - "False", - "False" - ], - "config__algorithm__sv_regions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - } - ], - "config__algorithm__svcaller": [ - [ - "cnvkit", - "manta", - "lumpy" - ], - [ - "cnvkit", - "manta", - "lumpy" - ] - ], - "config__algorithm__svprioritize": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/automated/variant_regions-bam.bed" - } - ], - "config__algorithm__svvalidate": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/reference_material/Test1-grade.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/reference_material/Test1-grade.vcf.gz.tbi" - } - ] - }, - null - ], - "config__algorithm__tools_off": [ - [], - [] - ], - "config__algorithm__tools_on": [ - [ - "coverage_perbase" - ], - [ - "coverage_perbase" - ] - ], - "config__algorithm__trim_reads": [ - "False", - "False" - ], - "config__algorithm__validate": [ - null, - null - ], - "config__algorithm__validate_regions": [ - null, - null - ], - "config__algorithm__variant_regions": [ - null, - null - ], - "config__algorithm__variantcaller": [ - [ - "vardict" - ], - [ - "vardict" - ] - ], - "config__algorithm__vcfanno": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.conf" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.lua" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/somatic.conf" - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.conf" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/config/vcfanno/gemini.lua" - } - ] - ], - "description": [ - "Test1", - "Test2" - ], - "files": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam.bai" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/100326_FC6107FAAXX/7_100326_FC6107FAAXX.bam.bai" - } - ] - } - ] - ], - "genome_build": [ - "hg19", - "hg19" - ], - "genome_resources__aliases__ensembl": [ - "homo_sapiens_vep_83_GRCh37", - "homo_sapiens_vep_83_GRCh37" - ], - "genome_resources__aliases__human": [ - "True", - "True" - ], - "genome_resources__aliases__snpeff": [ - "hg19", - "hg19" - ], - "genome_resources__rnaseq__gene_bed": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rnaseq/ref-transcripts.bed" - } - ], - "genome_resources__variation__1000g": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/1000g.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__clinvar": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/clinvar.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__cosmic": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/cosmic.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__dbsnp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/dbsnp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__encode_blacklist": [ - null, - null - ], - "genome_resources__variation__esp": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/esp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__exac": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/exac.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__gc_profile": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/gc/GC_profile.1000bp.cnp" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/gc/GC_profile.1000bp.cnp" - } - ], - "genome_resources__variation__germline_het_pon": [ - null, - null - ], - "genome_resources__variation__gnomad_exome": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/gnomad_exome.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__lcr": [ - null, - null - ], - "genome_resources__variation__polyx": [ - null, - null - ], - "genome_resources__variation__train_hapmap": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__train_indels": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/variation/Mills_Devine_2hit.indels.vcf.gz.tbi" - } - ] - } - ], - "metadata__batch": [ - "b1", - "b1" - ], - "metadata__phenotype": [ - "tumor", - "normal" - ], - "reference__bwa__indexes": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.sa" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/bwa/hg19.fa.sa" - } - ] - } - ], - "reference__fasta__base": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/seq/hg19.dict" - } - ] - } - ], - "reference__genome_context": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/coverage/problem_regions/GA4GH/test2.bed.gz.tbi" - } - ] - } - ] - ], - "reference__rtg": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/rtg--hg19.sdf-wf.tar.gz" - } - ], - "reference__snpeff__hg19": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/snpeff--hg19-wf.tar.gz" - } - ], - "reference__versions": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/versions.csv" - } - ], - "reference__viral": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.amb" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.sa" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.bwt" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.amb" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.sa" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/test_bcbio_cwl/testdata/genomes/hg19/viral/gdc-viral.fa.bwt" - } - ] - } - ], - "resources": [ - "{}", - "{}" - ], - "rgnames__lane": [ - "Test1", - "Test2" - ], - "rgnames__lb": [ - null, - null - ], - "rgnames__pl": [ - "illumina", - "illumina" - ], - "rgnames__pu": [ - "Test1", - "Test2" - ], - "rgnames__rg": [ - "Test1", - "Test2" - ], - "rgnames__sample": [ - "Test1", - "Test2" - ], - "vrn_file": [ - null, - null - ] -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/main-svcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/main-svcall.cwl deleted file mode 100644 index ec310cc54cd..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/main-svcall.cwl +++ /dev/null @@ -1,975 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: files - secondaryFiles: - - .bai - type: - items: - items: File - type: array - type: array -- id: config__algorithm__trim_reads - type: - items: - - string - - 'null' - - boolean - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: config__algorithm__vcfanno - type: - items: - items: File - type: array - type: array -- id: config__algorithm__svprioritize - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__adapters - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__gc_profile - type: - items: File - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__versions - type: - items: File - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__validate - type: - items: - - 'null' - - string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: description - type: - items: string - type: array -- id: config__algorithm__validate_regions - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: genome_resources__variation__germline_het_pon - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: reference__viral - secondaryFiles: - - .amb - - .ann - - .sa - - .pac - - ^.dict - - .bwt - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: config__algorithm__sv_regions - type: - items: File - type: array -- id: analysis - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__svvalidate - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__rtg - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -outputs: -- id: rgnames__sample_out - outputSource: prep_samples/rgnames__sample - type: - items: string - type: array -- id: align_bam - outputSource: postprocess_alignment/align_bam - type: - items: - - File - - 'null' - type: array -- id: regions__sample_callable - outputSource: postprocess_alignment/regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: validate__grading_summary - outputSource: summarize_vc/validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: variants__calls - outputSource: summarize_vc/variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - outputSource: summarize_vc/variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: sv__calls - outputSource: summarize_sv/sv__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: svvalidate__grading_summary - outputSource: summarize_sv/svvalidate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: sv__prioritize__tsv - outputSource: summarize_sv/sv__prioritize__tsv - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__prioritize__raw - outputSource: summarize_sv/sv__prioritize__raw - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__supplemental - outputSource: summarize_sv/sv__supplemental - type: - items: - items: - - File - type: array - type: array -- id: summary__multiqc - outputSource: multiqc_summary/summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - outputSource: multiqc_summary/versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - outputSource: multiqc_summary/versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: alignment_to_rec - in: - - id: files - source: files - - id: analysis - source: analysis - - id: config__algorithm__align_split_size - source: config__algorithm__align_split_size - - id: reference__fasta__base - source: reference__fasta__base - - id: rgnames__pl - source: rgnames__pl - - id: rgnames__sample - source: rgnames__sample - - id: rgnames__pu - source: rgnames__pu - - id: rgnames__lane - source: rgnames__lane - - id: rgnames__rg - source: rgnames__rg - - id: rgnames__lb - source: rgnames__lb - - id: reference__bwa__indexes - source: reference__bwa__indexes - - id: config__algorithm__aligner - source: config__algorithm__aligner - - id: config__algorithm__trim_reads - source: config__algorithm__trim_reads - - id: config__algorithm__adapters - source: config__algorithm__adapters - - id: config__algorithm__bam_clean - source: config__algorithm__bam_clean - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: config__algorithm__mark_duplicates - source: config__algorithm__mark_duplicates - - id: resources - source: resources - - id: description - source: description - out: - - id: alignment_rec - run: steps/alignment_to_rec.cwl -- id: alignment - in: - - id: alignment_rec - source: alignment_to_rec/alignment_rec - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: wf-alignment.cwl - scatter: - - alignment_rec - scatterMethod: dotproduct -- id: prep_samples_to_rec - in: - - id: rgnames__sample - source: rgnames__sample - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: prep_samples_rec - run: steps/prep_samples_to_rec.cwl -- id: prep_samples - in: - - id: prep_samples_rec - source: prep_samples_to_rec/prep_samples_rec - out: - - id: rgnames__sample - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - run: steps/prep_samples.cwl - scatter: - - prep_samples_rec - scatterMethod: dotproduct -- id: postprocess_alignment_to_rec - in: - - id: align_bam - source: alignment/align_bam - - id: config__algorithm__coverage_interval - source: config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: prep_samples/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: prep_samples/config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - source: prep_samples/config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - source: prep_samples/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: prep_samples/config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - source: prep_samples/config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - source: prep_samples/config__algorithm__seq2c_bed_ready - - id: config__algorithm__recalibrate - source: config__algorithm__recalibrate - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: postprocess_alignment_rec - run: steps/postprocess_alignment_to_rec.cwl -- id: postprocess_alignment - in: - - id: postprocess_alignment_rec - source: postprocess_alignment_to_rec/postprocess_alignment_rec - out: - - id: config__algorithm__coverage_interval - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - - id: regions__callable - - id: regions__sample_callable - - id: regions__nblock - - id: depth__samtools__stats - - id: depth__samtools__idxstats - - id: depth__variant_regions__regions - - id: depth__variant_regions__dist - - id: depth__sv_regions__regions - - id: depth__sv_regions__dist - - id: depth__coverage__regions - - id: depth__coverage__dist - - id: depth__coverage__thresholds - - id: align_bam - run: steps/postprocess_alignment.cwl - scatter: - - postprocess_alignment_rec - scatterMethod: dotproduct -- id: combine_sample_regions - in: - - id: regions__callable - source: postprocess_alignment/regions__callable - - id: regions__nblock - source: postprocess_alignment/regions__nblock - - id: metadata__batch - source: metadata__batch - - id: config__algorithm__nomap_split_size - source: config__algorithm__nomap_split_size - - id: config__algorithm__nomap_split_targets - source: config__algorithm__nomap_split_targets - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: config__algorithm__callable_regions - - id: config__algorithm__non_callable_regions - - id: config__algorithm__callable_count - run: steps/combine_sample_regions.cwl -- id: batch_for_variantcall - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: align_bam - source: postprocess_alignment/align_bam - - id: vrn_file - source: vrn_file - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: config__algorithm__variantcaller - source: config__algorithm__variantcaller - - id: config__algorithm__ensemble - source: config__algorithm__ensemble - - id: config__algorithm__vcfanno - source: config__algorithm__vcfanno - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__effects - source: config__algorithm__effects - - id: config__algorithm__min_allele_fraction - source: config__algorithm__min_allele_fraction - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__validate - source: config__algorithm__validate - - id: config__algorithm__validate_regions - source: config__algorithm__validate_regions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__rtg - source: reference__rtg - - id: reference__genome_context - source: reference__genome_context - - id: genome_resources__variation__clinvar - source: genome_resources__variation__clinvar - - id: genome_resources__variation__cosmic - source: genome_resources__variation__cosmic - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__esp - source: genome_resources__variation__esp - - id: genome_resources__variation__exac - source: genome_resources__variation__exac - - id: genome_resources__variation__gnomad_exome - source: genome_resources__variation__gnomad_exome - - id: genome_resources__variation__1000g - source: genome_resources__variation__1000g - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__aliases__ensembl - source: genome_resources__aliases__ensembl - - id: genome_resources__aliases__human - source: genome_resources__aliases__human - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__hg19 - source: reference__snpeff__hg19 - - id: genome_resources__variation__train_hapmap - source: genome_resources__variation__train_hapmap - - id: genome_resources__variation__train_indels - source: genome_resources__variation__train_indels - - id: resources - source: resources - - id: description - source: description - out: - - id: batch_rec - run: steps/batch_for_variantcall.cwl -- id: variantcall - in: - - id: batch_rec - source: batch_for_variantcall/batch_rec - out: - - id: vc_rec - run: wf-variantcall.cwl - scatter: - - batch_rec - scatterMethod: dotproduct -- id: summarize_vc - in: - - id: vc_rec - source: variantcall/vc_rec - out: - - id: variants__calls - - id: variants__gvcf - - id: variants__samples - - id: validate__grading_summary - - id: validate__grading_plots - run: steps/summarize_vc.cwl -- id: calculate_sv_bins - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: reference__fasta__base - source: reference__fasta__base - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__sv_regions - source: config__algorithm__sv_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__svcaller - source: config__algorithm__svcaller - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: resources - source: resources - - id: description - source: description - out: - - id: sv_bin_rec - run: steps/calculate_sv_bins.cwl -- id: calculate_sv_coverage - in: - - id: sv_bin_rec - source: calculate_sv_bins/sv_bin_rec - out: - - id: sv_rawcoverage_rec - run: steps/calculate_sv_coverage.cwl - scatter: - - sv_bin_rec - scatterMethod: dotproduct -- id: normalize_sv_coverage - in: - - id: sv_rawcoverage_rec - source: calculate_sv_coverage/sv_rawcoverage_rec - out: - - id: sv_coverage_rec - run: steps/normalize_sv_coverage.cwl -- id: batch_for_sv - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: work_bam_plus__disc - source: alignment/work_bam_plus__disc - - id: work_bam_plus__sr - source: alignment/work_bam_plus__sr - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: config__algorithm__svprioritize - source: config__algorithm__svprioritize - - id: config__algorithm__svvalidate - source: config__algorithm__svvalidate - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: genome_resources__variation__gc_profile - source: genome_resources__variation__gc_profile - - id: genome_resources__variation__germline_het_pon - source: genome_resources__variation__germline_het_pon - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__hg19 - source: reference__snpeff__hg19 - - id: sv_coverage_rec - source: normalize_sv_coverage/sv_coverage_rec - - id: variants__samples - source: summarize_vc/variants__samples - out: - - id: sv_batch_rec - run: steps/batch_for_sv.cwl -- id: svcall - in: - - id: sv_batch_rec - source: batch_for_sv/sv_batch_rec - out: - - id: sv_rec - run: wf-svcall.cwl - scatter: - - sv_batch_rec - scatterMethod: dotproduct -- id: summarize_sv - in: - - id: sv_rec - source: svcall/sv_rec - out: - - id: sv__calls - - id: sv__supplemental - - id: sv__prioritize__tsv - - id: sv__prioritize__raw - - id: svvalidate__grading_summary - - id: svvalidate__grading_plots - run: steps/summarize_sv.cwl -- id: qc_to_rec - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: analysis - source: analysis - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__versions - source: reference__versions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: genome_build - source: genome_build - - id: config__algorithm__qc - source: config__algorithm__qc - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: depth__variant_regions__dist - source: postprocess_alignment/depth__variant_regions__dist - - id: depth__samtools__stats - source: postprocess_alignment/depth__samtools__stats - - id: depth__samtools__idxstats - source: postprocess_alignment/depth__samtools__idxstats - - id: depth__sv_regions__regions - source: postprocess_alignment/depth__sv_regions__regions - - id: depth__sv_regions__dist - source: postprocess_alignment/depth__sv_regions__dist - - id: depth__coverage__regions - source: postprocess_alignment/depth__coverage__regions - - id: depth__coverage__dist - source: postprocess_alignment/depth__coverage__dist - - id: depth__coverage__thresholds - source: postprocess_alignment/depth__coverage__thresholds - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__coverage - source: postprocess_alignment/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: postprocess_alignment/config__algorithm__coverage_merged - - id: variants__samples - source: summarize_vc/variants__samples - - id: reference__viral - source: reference__viral - - id: resources - source: resources - - id: description - source: description - out: - - id: qc_rec - run: steps/qc_to_rec.cwl -- id: pipeline_summary - in: - - id: qc_rec - source: qc_to_rec/qc_rec - out: - - id: qcout_rec - run: steps/pipeline_summary.cwl - scatter: - - qc_rec - scatterMethod: dotproduct -- id: multiqc_summary - in: - - id: qcout_rec - source: pipeline_summary/qcout_rec - out: - - id: summary__multiqc - - id: versions__tools - - id: versions__data - run: steps/multiqc_summary.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/alignment_to_rec.cwl deleted file mode 100644 index 8208a1f1eed..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/alignment_to_rec.cwl +++ /dev/null @@ -1,206 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=alignment_rec:resources;description;config__algorithm__align_split_size;files;config__algorithm__trim_reads;reference__fasta__base;config__algorithm__adapters;rgnames__lb;rgnames__rg;rgnames__lane;reference__bwa__indexes;config__algorithm__bam_clean;config__algorithm__aligner;rgnames__pl;rgnames__pu;config__algorithm__mark_duplicates;analysis;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=files:var,analysis:var,config__algorithm__align_split_size:var,reference__fasta__base:var,rgnames__pl:var,rgnames__sample:var,rgnames__pu:var,rgnames__lane:var,rgnames__rg:var,rgnames__lb:var,reference__bwa__indexes:var,config__algorithm__aligner:var,config__algorithm__trim_reads:var,config__algorithm__adapters:var,config__algorithm__bam_clean:var,config__algorithm__variant_regions:var,config__algorithm__mark_duplicates:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: files - secondaryFiles: - - .bai - type: - items: - items: File - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: config__algorithm__trim_reads - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__adapters - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__variant_regions - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: - - 'null' - - string - name: alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/batch_for_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/batch_for_sv.cwl deleted file mode 100644 index 5c725369def..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/batch_for_sv.cwl +++ /dev/null @@ -1,359 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=sv_batch_rec:resources;description;config__algorithm__svprioritize;genome_resources__variation__gc_profile;reference__snpeff__hg19;genome_build;genome_resources__variation__germline_het_pon;config__algorithm__tools_off;analysis;config__algorithm__tools_on;config__algorithm__svvalidate;genome_resources__aliases__snpeff;work_bam_plus__disc;work_bam_plus__sr;regions__sample_callable;variants__samples;depth__bins__normalized;depth__bins__background;depth__bins__target;depth__bins__antitarget;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__sv_regions;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,work_bam_plus__disc:var,work_bam_plus__sr:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,config__algorithm__svprioritize:var,config__algorithm__svvalidate:var,regions__sample_callable:var,genome_resources__variation__gc_profile:var,genome_resources__variation__germline_het_pon:var,genome_resources__aliases__snpeff:var,reference__snpeff__hg19:var,sv_coverage_rec:record,variants__samples:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1024 - ramMin: 6144 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__svprioritize - type: - items: File - type: array -- id: config__algorithm__svvalidate - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: genome_resources__variation__gc_profile - type: - items: File - type: array -- id: genome_resources__variation__germline_het_pon - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: sv_coverage_rec - type: - items: - fields: - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_coverage_rec - type: record - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -outputs: -- id: sv_batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: File - - name: genome_resources__variation__gc_profile - type: File - - name: reference__snpeff__hg19 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - File - - 'null' - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/batch_for_variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/batch_for_variantcall.cwl deleted file mode 100644 index 1a04f1e6887..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/batch_for_variantcall.cwl +++ /dev/null @@ -1,401 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=batch_rec:resources;description;reference__fasta__base;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;genome_resources__variation__1000g;config__algorithm__min_allele_fraction;vrn_file;genome_resources__variation__train_hapmap;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;metadata__phenotype;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;align_bam;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,align_bam:var,vrn_file:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,regions__sample_callable:var,config__algorithm__variantcaller:var,config__algorithm__ensemble:var,config__algorithm__vcfanno:var,config__algorithm__coverage_interval:var,config__algorithm__effects:var,config__algorithm__min_allele_fraction:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__validate:var,config__algorithm__validate_regions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,reference__fasta__base:var,reference__rtg:var,reference__genome_context:var,genome_resources__variation__clinvar:var,genome_resources__variation__cosmic:var,genome_resources__variation__dbsnp:var,genome_resources__variation__esp:var,genome_resources__variation__exac:var,genome_resources__variation__gnomad_exome:var,genome_resources__variation__1000g:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__aliases__ensembl:var,genome_resources__aliases__human:var,genome_resources__aliases__snpeff:var,reference__snpeff__hg19:var,genome_resources__variation__train_hapmap:var,genome_resources__variation__train_indels:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_variantcall -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__ensemble - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__vcfanno - type: - items: - items: File - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__validate - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__validate_regions - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__rtg - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__hg19 - type: - items: File - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/calculate_sv_bins.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/calculate_sv_bins.cwl deleted file mode 100644 index 34683452f0d..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/calculate_sv_bins.cwl +++ /dev/null @@ -1,221 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv_bin_rec:regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;resources;description;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__sv_regions;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=align_bam:var,reference__fasta__base:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__sv_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__svcaller:var,depth__variant_regions__regions:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__rnaseq__gene_bed:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- calculate_sv_bins -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__sv_regions - type: - items: File - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: sv_bin_rec - type: - items: - fields: - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_bin_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/calculate_sv_coverage.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/calculate_sv_coverage.cwl deleted file mode 100644 index 79ba8e7eaf5..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/calculate_sv_coverage.cwl +++ /dev/null @@ -1,207 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=sv_rawcoverage_rec:depth__bins__target;depth__bins__antitarget;resources;description;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__sv_regions;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_bin_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- calculate_sv_coverage -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit -inputs: -- id: sv_bin_rec - type: - fields: - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_bin_rec - type: record -outputs: -- id: sv_rawcoverage_rec - type: - fields: - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rawcoverage_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/combine_sample_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/combine_sample_regions.cwl deleted file mode 100644 index 434161e89da..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/combine_sample_regions.cwl +++ /dev/null @@ -1,99 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=config__algorithm__callable_regions,config__algorithm__non_callable_regions,config__algorithm__callable_count -- sentinel_inputs=regions__callable:var,regions__nblock:var,metadata__batch:var,config__algorithm__nomap_split_size:var,config__algorithm__nomap_split_targets:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- combine_sample_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: regions__callable - type: - items: - - File - - 'null' - type: array -- id: regions__nblock - type: - items: - - File - - 'null' - type: array -- id: metadata__batch - type: - items: string - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__non_callable_regions - type: - items: File - type: array -- id: config__algorithm__callable_count - type: - items: int - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/compare_to_rm.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/compare_to_rm.cwl deleted file mode 100644 index 142a0d44ce9..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/compare_to_rm.cwl +++ /dev/null @@ -1,307 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vc_rec:batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;resources;description;vrn_file;reference__fasta__base;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;metadata__batch;config__algorithm__min_allele_fraction;reference__genome_context;config__algorithm__validate;reference__snpeff__hg19;config__algorithm__validate_regions;genome_build;metadata__phenotype;genome_resources__aliases__human;config__algorithm__tools_off;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;reference__rtg;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- compare_to_rm -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: gvcf-regions - specs: - - https://anaconda.org/bioconda/gvcf-regions - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: rtg-tools - specs: - - https://anaconda.org/bioconda/rtg-tools - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file - secondaryFiles: - - .tbi - type: File -outputs: -- id: vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/concat_batch_variantcalls.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/concat_batch_variantcalls.cwl deleted file mode 100644 index b8cd24d68bc..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/concat_batch_variantcalls.cwl +++ /dev/null @@ -1,196 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-merge -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,region_block:var,vrn_file_region:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- concat_batch_variantcalls -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block - type: - items: - items: string - type: array - type: array -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/detect_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/detect_sv.cwl deleted file mode 100644 index 4aab2c1343b..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/detect_sv.cwl +++ /dev/null @@ -1,396 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=sv_rec:sv__variantcaller;sv__vrn_file;sv__supplemental;svvalidate__summary;resources;description;config__algorithm__svprioritize;genome_resources__variation__gc_profile;genome_build;genome_resources__variation__germline_het_pon;config__algorithm__tools_off;analysis;config__algorithm__tools_on;config__algorithm__svvalidate;genome_resources__aliases__snpeff;regions__sample_callable;variants__samples;depth__bins__normalized;depth__bins__background;depth__bins__target;depth__bins__antitarget;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__sv_regions;config__algorithm__variant_regions;config__algorithm__exclude_regions;config__algorithm__variant_regions_merged;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- detect_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit - - package: delly - specs: - - https://anaconda.org/bioconda/delly - - package: duphold - specs: - - https://anaconda.org/bioconda/duphold - - package: extract-sv-reads - specs: - - https://anaconda.org/bioconda/extract-sv-reads - - package: lumpy-sv - specs: - - https://anaconda.org/bioconda/lumpy-sv - - package: manta - specs: - - https://anaconda.org/bioconda/manta - - package: break-point-inspector - specs: - - https://anaconda.org/bioconda/break-point-inspector - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: smoove - specs: - - https://anaconda.org/bioconda/smoove - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: seq2c - specs: - - https://anaconda.org/bioconda/seq2c - - package: simple_sv_annotation - specs: - - https://anaconda.org/bioconda/simple_sv_annotation - - package: survivor - specs: - - https://anaconda.org/bioconda/survivor - - package: svtools - specs: - - https://anaconda.org/bioconda/svtools - - package: svtyper - specs: - - https://anaconda.org/bioconda/svtyper - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: xorg-libxt - specs: - - https://anaconda.org/bioconda/xorg-libxt - - package: vawk - specs: - - https://anaconda.org/bioconda/vawk -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: sv_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: File - - name: genome_resources__variation__gc_profile - type: File - - name: reference__snpeff__hg19 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - File - - 'null' - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array -outputs: -- id: sv_rec - type: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: File - - name: genome_resources__variation__gc_profile - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - File - - 'null' - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/get_parallel_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/get_parallel_regions.cwl deleted file mode 100644 index 71ac417a4ce..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/get_parallel_regions.cwl +++ /dev/null @@ -1,171 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-split -- sentinel_outputs=region_block -- sentinel_inputs=batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- get_parallel_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: region_block - type: - items: - items: string - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/merge_split_alignments.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/merge_split_alignments.cwl deleted file mode 100644 index afe6f73cfc2..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/merge_split_alignments.cwl +++ /dev/null @@ -1,171 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-merge -- sentinel_outputs=align_bam,work_bam_plus__disc,work_bam_plus__sr,hla__fastq -- sentinel_inputs=alignment_rec:record,work_bam:var,align_bam:var,work_bam_plus__disc:var,work_bam_plus__sr:var,hla__fastq:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- merge_split_alignments -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1035 - ramMin: 6144 - tmpdirMin: 6 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: - - 'null' - - string - name: alignment_rec - type: record -- id: work_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: align_bam_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__disc_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__sr_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: hla__fastq_toolinput - type: - items: - - 'null' - - items: File - type: array - type: array -outputs: -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/multiqc_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/multiqc_summary.cwl deleted file mode 100644 index 82ce9f97a43..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/multiqc_summary.cwl +++ /dev/null @@ -1,95 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=summary__multiqc,versions__tools,versions__data -- sentinel_inputs=qcout_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- multiqc_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1030 - ramMin: 3072 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: multiqc - specs: - - https://anaconda.org/bioconda/multiqc - - package: multiqc-bcbio - specs: - - https://anaconda.org/bioconda/multiqc-bcbio -inputs: -- id: qcout_rec - type: - items: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record - type: array -outputs: -- id: summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/normalize_sv_coverage.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/normalize_sv_coverage.cwl deleted file mode 100644 index 813a2a21a63..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/normalize_sv_coverage.cwl +++ /dev/null @@ -1,224 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv_coverage_rec:depth__bins__normalized;depth__bins__background;resources;description;depth__bins__target;depth__bins__antitarget;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__sv_regions;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_rawcoverage_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- normalize_sv_coverage -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit -inputs: -- id: sv_rawcoverage_rec - type: - items: - fields: - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rawcoverage_rec - type: record - type: array -outputs: -- id: sv_coverage_rec - type: - items: - fields: - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_coverage_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/pipeline_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/pipeline_summary.cwl deleted file mode 100644 index d320e1afb86..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/pipeline_summary.cwl +++ /dev/null @@ -1,221 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=qcout_rec:summary__qc;summary__metrics;description;reference__versions;genome_build;config__algorithm__tools_off;config__algorithm__qc;config__algorithm__tools_on -- sentinel_inputs=qc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- pipeline_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: fastqc=0.11.7=5 - specs: - - https://anaconda.org/bioconda/fastqc=0.11.7=5 - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: hts-nim-tools - specs: - - https://anaconda.org/bioconda/hts-nim-tools - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: qsignature - specs: - - https://anaconda.org/bioconda/qsignature - - package: qualimap - specs: - - https://anaconda.org/bioconda/qualimap - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: preseq - specs: - - https://anaconda.org/bioconda/preseq - - package: peddy - specs: - - https://anaconda.org/bioconda/peddy - - package: verifybamid2 - specs: - - https://anaconda.org/bioconda/verifybamid2 -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: qc_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: reference__viral - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record -outputs: -- id: qcout_rec - type: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_alignment.cwl deleted file mode 100644 index ccdfc8e177e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_alignment.cwl +++ /dev/null @@ -1,223 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=config__algorithm__coverage_interval,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready,regions__callable,regions__sample_callable,regions__nblock,depth__samtools__stats,depth__samtools__idxstats,depth__variant_regions__regions,depth__variant_regions__dist,depth__sv_regions__regions,depth__sv_regions__dist,depth__coverage__regions,depth__coverage__dist,depth__coverage__thresholds,align_bam -- sentinel_inputs=postprocess_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1033 - ramMin: 6144 - tmpdirMin: 5 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon -- class: arv:APIRequirement -inputs: -- id: postprocess_alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record -outputs: -- id: config__algorithm__coverage_interval - type: - - string - - 'null' -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -- id: regions__callable - type: - - File - - 'null' -- id: regions__sample_callable - type: - - File - - 'null' -- id: regions__nblock - type: - - File - - 'null' -- id: depth__samtools__stats - type: - - File - - 'null' -- id: depth__samtools__idxstats - type: - - File - - 'null' -- id: depth__variant_regions__regions - type: - - File - - 'null' -- id: depth__variant_regions__dist - type: - - File - - 'null' -- id: depth__sv_regions__regions - type: - - File - - 'null' -- id: depth__sv_regions__dist - type: - - File - - 'null' -- id: depth__coverage__regions - type: - - File - - 'null' -- id: depth__coverage__dist - type: - - File - - 'null' -- id: depth__coverage__thresholds - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_alignment_to_rec.cwl deleted file mode 100644 index 1f033c9d3c7..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_alignment_to_rec.cwl +++ /dev/null @@ -1,233 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=postprocess_alignment_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;genome_resources__variation__lcr;config__algorithm__recalibrate;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__tools_on;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__variant_regions_orig;config__algorithm__coverage;config__algorithm__coverage_merged;config__algorithm__coverage_orig;config__algorithm__seq2c_bed_ready -- sentinel_inputs=align_bam:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__variant_regions_orig:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,config__algorithm__coverage_orig:var,config__algorithm__seq2c_bed_ready:var,config__algorithm__recalibrate:var,config__algorithm__tools_on:var,genome_resources__rnaseq__gene_bed:var,genome_resources__variation__dbsnp:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__seq2c_bed_ready - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__polyx - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__encode_blacklist - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: postprocess_alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_variants.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_variants.cwl deleted file mode 100644 index dd40cf1d9a4..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/postprocess_variants.cwl +++ /dev/null @@ -1,180 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_variants -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1025 - ramMin: 6144 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: snpeff - specs: - - https://anaconda.org/bioconda/snpeff - version: - - 4.3.1t -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file_toolinput - secondaryFiles: - - .tbi - type: File -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_align_inputs.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_align_inputs.cwl deleted file mode 100644 index cd325c5f774..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_align_inputs.cwl +++ /dev/null @@ -1,145 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-split -- sentinel_outputs=process_alignment_rec:files;config__algorithm__quality_format;align_split -- sentinel_inputs=alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_align_inputs -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1028 - ramMin: 6144 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 4 -- class: SoftwareRequirement - packages: - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: atropos;env - specs: - - https://anaconda.org/bioconda/atropos;env - version: - - python3 - - package: optitype - specs: - - https://anaconda.org/bioconda/optitype - - package: razers3 - specs: - - https://anaconda.org/bioconda/razers3 - version: - - 3.5.0 - - package: coincbc - specs: - - https://anaconda.org/bioconda/coincbc -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: - - 'null' - - string - name: alignment_rec - type: record -outputs: -- id: process_alignment_rec - type: - items: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_samples.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_samples.cwl deleted file mode 100644 index 63d24367261..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_samples.cwl +++ /dev/null @@ -1,93 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=rgnames__sample,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready -- sentinel_inputs=prep_samples_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy -inputs: -- id: prep_samples_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: - - 'null' - - string - name: prep_samples_rec - type: record -outputs: -- id: rgnames__sample - type: string -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_samples_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_samples_to_rec.cwl deleted file mode 100644 index da7b1d7f2b0..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/prep_samples_to_rec.cwl +++ /dev/null @@ -1,79 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=prep_samples_rec:resources;description;reference__fasta__base;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=rgnames__sample:var,config__algorithm__variant_regions:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: prep_samples_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: - - 'null' - - string - name: prep_samples_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/process_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/process_alignment.cwl deleted file mode 100644 index af6768e2f88..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/process_alignment.cwl +++ /dev/null @@ -1,201 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-parallel -- sentinel_outputs=work_bam,align_bam,hla__fastq,work_bam_plus__disc,work_bam_plus__sr -- sentinel_inputs=alignment_rec:record,process_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- process_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 7 -- class: SoftwareRequirement - packages: - - package: bwa - specs: - - https://anaconda.org/bioconda/bwa - - package: bwakit - specs: - - https://anaconda.org/bioconda/bwakit - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: minimap2 - specs: - - https://anaconda.org/bioconda/minimap2 - - package: novoalign - specs: - - https://anaconda.org/bioconda/novoalign - - package: snap-aligner - specs: - - https://anaconda.org/bioconda/snap-aligner - version: - - 1.0dev.97 - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: fgbio - specs: - - https://anaconda.org/bioconda/fgbio - - package: umis - specs: - - https://anaconda.org/bioconda/umis - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: seqtk - specs: - - https://anaconda.org/bioconda/seqtk - - package: samblaster - specs: - - https://anaconda.org/bioconda/samblaster - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -- class: arv:APIRequirement -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: - - 'null' - - string - name: alignment_rec - type: record -- id: process_alignment_rec - type: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record -outputs: -- id: work_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/qc_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/qc_to_rec.cwl deleted file mode 100644 index 4565b513de7..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/qc_to_rec.cwl +++ /dev/null @@ -1,308 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=qc_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;metadata__batch;reference__versions;genome_build;metadata__phenotype;config__algorithm__tools_off;reference__viral;config__algorithm__qc;analysis;config__algorithm__tools_on;config__algorithm__variant_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__coverage;config__algorithm__coverage_merged;depth__samtools__stats;depth__samtools__idxstats;depth__variant_regions__regions;depth__variant_regions__dist;depth__sv_regions__regions;depth__sv_regions__dist;depth__coverage__regions;depth__coverage__dist;depth__coverage__thresholds;variants__samples -- sentinel_inputs=align_bam:var,analysis:var,reference__fasta__base:var,reference__versions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,genome_build:var,config__algorithm__qc:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__coverage_interval:var,depth__variant_regions__regions:var,depth__variant_regions__dist:var,depth__samtools__stats:var,depth__samtools__idxstats:var,depth__sv_regions__regions:var,depth__sv_regions__dist:var,depth__coverage__regions:var,depth__coverage__dist:var,depth__coverage__thresholds:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,variants__samples:var,reference__viral:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- qc_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1028 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: analysis - type: - items: string - type: array -- id: reference__fasta__base - secondaryFiles: - - .fai - - ^.dict - type: - items: File - type: array -- id: reference__versions - type: - items: File - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_build - type: - items: string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__variant_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__stats - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__idxstats - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__regions - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__thresholds - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: reference__viral - secondaryFiles: - - .amb - - .ann - - .sa - - .pac - - ^.dict - - .bwt - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: qc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: reference__versions - type: File - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: reference__viral - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/summarize_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/summarize_sv.cwl deleted file mode 100644 index 87f503d9117..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/summarize_sv.cwl +++ /dev/null @@ -1,228 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv__calls,sv__supplemental,sv__prioritize__tsv,sv__prioritize__raw,svvalidate__grading_summary,svvalidate__grading_plots -- sentinel_inputs=sv_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1027 - ramMin: 3072 - tmpdirMin: 2 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: sv_rec - type: - items: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: File - - name: genome_resources__variation__gc_profile - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - File - - 'null' - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array - type: array -outputs: -- id: sv__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__supplemental - type: - items: - items: - - File - type: array - type: array -- id: sv__prioritize__tsv - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__prioritize__raw - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: svvalidate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: svvalidate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/summarize_vc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/summarize_vc.cwl deleted file mode 100644 index c560affeded..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/summarize_vc.cwl +++ /dev/null @@ -1,193 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=variants__calls,variants__gvcf,variants__samples,validate__grading_summary,validate__grading_plots -- sentinel_inputs=vc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_vc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1025 - ramMin: 3072 - tmpdirMin: 1 -- class: dx:InputResourceRequirement - indirMin: 1 -inputs: -- id: vc_rec - type: - items: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array - type: array -outputs: -- id: variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: validate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/variantcall_batch_region.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/variantcall_batch_region.cwl deleted file mode 100644 index 7e8f1574bfc..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/steps/variantcall_batch_region.cwl +++ /dev/null @@ -1,266 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-parallel -- sentinel_outputs=vrn_file_region,region_block -- sentinel_inputs=batch_rec:record,region_block:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- variantcall_batch_region -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 2 - outdirMin: 1030 - ramMin: 6144 - tmpdirMin: 3 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: freebayes - specs: - - https://anaconda.org/bioconda/freebayes - version: - - 1.1.0.46 - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: vqsr_cnn - specs: - - https://anaconda.org/bioconda/vqsr_cnn - - package: deepvariant - specs: - - https://anaconda.org/bioconda/deepvariant - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: octopus - specs: - - https://anaconda.org/bioconda/octopus - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: platypus-variant - specs: - - https://anaconda.org/bioconda/platypus-variant - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: strelka - specs: - - https://anaconda.org/bioconda/strelka - - package: vardict - specs: - - https://anaconda.org/bioconda/vardict - - package: vardict-java - specs: - - https://anaconda.org/bioconda/vardict-java - - package: varscan - specs: - - https://anaconda.org/bioconda/varscan - - package: moreutils - specs: - - https://anaconda.org/bioconda/moreutils - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno - - package: vcflib - specs: - - https://anaconda.org/bioconda/vcflib - - package: vt - specs: - - https://anaconda.org/bioconda/vt - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: perl - specs: - - https://anaconda.org/bioconda/perl -- class: arv:APIRequirement -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block_toolinput - type: - items: string - type: array -outputs: -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - - File - - 'null' -- id: region_block - type: - items: string - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-alignment.cwl deleted file mode 100644 index 3785183a8df..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-alignment.cwl +++ /dev/null @@ -1,146 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: - - string - - 'null' - - boolean - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - - 'null' - - items: 'null' - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: rgnames__pu - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: - - 'null' - - string - name: alignment_rec - type: record -outputs: -- id: align_bam - outputSource: merge_split_alignments/align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - outputSource: merge_split_alignments/work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - outputSource: merge_split_alignments/work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - outputSource: merge_split_alignments/hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: prep_align_inputs - in: - - id: alignment_rec - source: alignment_rec - out: - - id: process_alignment_rec - run: steps/prep_align_inputs.cwl -- id: process_alignment - in: - - id: alignment_rec - source: alignment_rec - - id: process_alignment_rec - source: prep_align_inputs/process_alignment_rec - out: - - id: work_bam - - id: align_bam - - id: hla__fastq - - id: work_bam_plus__disc - - id: work_bam_plus__sr - run: steps/process_alignment.cwl - scatter: - - process_alignment_rec - scatterMethod: dotproduct -- id: merge_split_alignments - in: - - id: alignment_rec - source: alignment_rec - - id: work_bam - source: process_alignment/work_bam - - id: align_bam_toolinput - source: process_alignment/align_bam - - id: work_bam_plus__disc_toolinput - source: process_alignment/work_bam_plus__disc - - id: work_bam_plus__sr_toolinput - source: process_alignment/work_bam_plus__sr - - id: hla__fastq_toolinput - source: process_alignment/hla__fastq - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: steps/merge_split_alignments.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-svcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-svcall.cwl deleted file mode 100644 index 01ae7b07f7a..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-svcall.cwl +++ /dev/null @@ -1,310 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: sv_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: File - - name: genome_resources__variation__gc_profile - type: File - - name: reference__snpeff__hg19 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - File - - 'null' - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array -outputs: -- id: sv_rec - outputSource: detect_sv/sv_rec - type: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: File - - name: genome_resources__variation__gc_profile - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - File - - 'null' - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: config__algorithm__sv_regions - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: detect_sv - in: - - id: sv_batch_rec - source: sv_batch_rec - out: - - id: sv_rec - run: steps/detect_sv.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-variantcall.cwl deleted file mode 100644 index f7dd3955627..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/svcall-workflow/wf-variantcall.cwl +++ /dev/null @@ -1,308 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: - - 'null' - - string - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: - - 'null' - - string - - name: genome_resources__variation__encode_blacklist - type: - - 'null' - - string - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: vc_rec - outputSource: compare_to_rm/vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - items: File - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__genome_context - type: - items: File - type: array - - name: config__algorithm__validate - type: - - 'null' - - string - - name: reference__snpeff__hg19 - type: File - - name: config__algorithm__validate_regions - type: - - 'null' - - string - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__ensemble - type: - - 'null' - - string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: reference__rtg - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: get_parallel_regions - in: - - id: batch_rec - source: batch_rec - out: - - id: region_block - run: steps/get_parallel_regions.cwl -- id: variantcall_batch_region - in: - - id: batch_rec - source: batch_rec - - id: region_block_toolinput - source: get_parallel_regions/region_block - out: - - id: vrn_file_region - - id: region_block - run: steps/variantcall_batch_region.cwl - scatter: - - region_block_toolinput - scatterMethod: dotproduct -- id: concat_batch_variantcalls - in: - - id: batch_rec - source: batch_rec - - id: region_block - source: variantcall_batch_region/region_block - - id: vrn_file_region - source: variantcall_batch_region/vrn_file_region - out: - - id: vrn_file - run: steps/concat_batch_variantcalls.cwl -- id: postprocess_variants - in: - - id: batch_rec - source: batch_rec - - id: vrn_file_toolinput - source: concat_batch_variantcalls/vrn_file - out: - - id: vrn_file - run: steps/postprocess_variants.cwl -- id: compare_to_rm - in: - - id: batch_rec - source: batch_rec - - id: vrn_file - source: postprocess_variants/vrn_file - out: - - id: vc_rec - run: steps/compare_to_rm.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test-samples.json b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test-samples.json deleted file mode 100644 index 1439ccade50..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test-samples.json +++ /dev/null @@ -1,1059 +0,0 @@ -{ - "expected_validation": - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/grading-summary-combined-expected.csv" - }, - "analysis": [ - "variant2", - "variant2" - ], - "config__algorithm__adapters": [ - [ - "polyx" - ], - [ - "polyx" - ] - ], - "config__algorithm__align_split_size": [ - null, - null - ], - "config__algorithm__aligner": [ - "bwa", - "bwa" - ], - "config__algorithm__archive": [ - null, - null - ], - "config__algorithm__bam_clean": [ - "False", - "False" - ], - "config__algorithm__coverage_interval": [ - null, - null - ], - "config__algorithm__effects": [ - "snpeff", - "snpeff" - ], - "config__algorithm__ensemble": [ - "{\"numpass\": 2}", - "{\"numpass\": 2}" - ], - "config__algorithm__exclude_regions": [ - [], - [] - ], - "config__algorithm__mark_duplicates": [ - "True", - "True" - ], - "config__algorithm__min_allele_fraction": [ - 10.0, - 10.0 - ], - "config__algorithm__nomap_split_size": [ - 250, - 250 - ], - "config__algorithm__nomap_split_targets": [ - 20, - 20 - ], - "config__algorithm__qc": [ - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "samtools", - "variants", - "viral" - ], - [ - "contamination", - "coverage", - "fastqc", - "peddy", - "picard", - "qsignature", - "samtools", - "variants", - "viral" - ] - ], - "config__algorithm__recalibrate": [ - "False", - "False" - ], - "config__algorithm__svcaller": [ - [ - "cnvkit" - ], - [ - "cnvkit" - ] - ], - "config__algorithm__svprioritize": [ - null, - null - ], - "config__algorithm__svvalidate": [ - null, - null - ], - "config__algorithm__tools_off": [ - [ - "gemini" - ], - [ - "gemini" - ] - ], - "config__algorithm__tools_on": [ - [ - "gatk4", - "break-point-inspector", - "noalt_calling" - ], - [ - "gatk4", - "break-point-inspector", - "noalt_calling" - ] - ], - "config__algorithm__trim_reads": [ - "atropos", - "atropos" - ], - "config__algorithm__validate": [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/wes-ensemble-annotated_exome_chr21_noAnnot_somatic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/wes-ensemble-annotated_exome_chr21_noAnnot_somatic.vcf.gz.tbi" - } - ] - }, - null - ], - "config__algorithm__validate_regions": [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/Exome-Agilent_V6_chr21.bed" - }, - null - ], - "config__algorithm__variant_regions": [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/Exome-Agilent_V6_chr21.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/Exome-Agilent_V6_chr21.bed" - } - ], - "config__algorithm__variantcaller": [ - [ - "germline:vardict", - "germline:strelka2", - "germline:gatk-haplotype", - "somatic:vardict", - "somatic:strelka2", - "somatic:mutect2" - ], - [ - "germline:vardict", - "germline:strelka2", - "germline:gatk-haplotype", - "somatic:vardict", - "somatic:strelka2", - "somatic:mutect2" - ] - ], - "config__algorithm__vcfanno": [ - [], - [] - ], - "description": [ - "NA12878_chr21", - "NA24385_chr21" - ], - "files": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/NA12878-ready_exome21.bam", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/NA12878-ready_exome21.bam.bai" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/NA24385-ready_exome21.bam", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/wes_agha_test/NA24385-ready_exome21.bam.bai" - } - ] - } - ] - ], - "genome_build": [ - "GRCh37", - "GRCh37" - ], - "genome_resources__aliases__ensembl": [ - "homo_sapiens_merged_vep_93_GRCh37", - "homo_sapiens_merged_vep_93_GRCh37" - ], - "genome_resources__aliases__human": [ - "True", - "True" - ], - "genome_resources__aliases__snpeff": [ - "GRCh37.75", - "GRCh37.75" - ], - "genome_resources__rnaseq__gene_bed": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/rnaseq/ref-transcripts.bed" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/rnaseq/ref-transcripts.bed" - } - ], - "genome_resources__variation__1000g": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/1000g.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/1000g.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/1000g.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__clinvar": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/clinvar.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/clinvar.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/clinvar.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__cosmic": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/cosmic.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/cosmic.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/cosmic.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__dbsnp": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/dbsnp-151.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/dbsnp-151.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/dbsnp-151.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/dbsnp-151.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__encode_blacklist": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi" - } - ] - } - ], - "genome_resources__variation__esp": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/esp.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/esp.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/esp.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__exac": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/exac.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/exac.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/exac.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__gc_profile": [ - null, - null - ], - "genome_resources__variation__germline_het_pon": [ - null, - null - ], - "genome_resources__variation__gnomad_exome": [ - null, - null - ], - "genome_resources__variation__lcr": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz.tbi" - } - ] - } - ], - "genome_resources__variation__polyx": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz.tbi" - } - ] - } - ], - "genome_resources__variation__train_hapmap": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/hapmap_3.3.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/hapmap_3.3.vcf.gz.tbi" - } - ] - } - ], - "genome_resources__variation__train_indels": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/Mills_and_1000G_gold_standard.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/Mills_and_1000G_gold_standard.indels.vcf.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/Mills_and_1000G_gold_standard.indels.vcf.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/variation/Mills_and_1000G_gold_standard.indels.vcf.gz.tbi" - } - ] - } - ], - "metadata__batch": [ - "wes", - "wes" - ], - "metadata__phenotype": [ - "tumor", - "normal" - ], - "reference__bwa__indexes": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.sa" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.amb", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/bwa/GRCh37.fa.sa" - } - ] - } - ], - "reference__fasta__base": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/seq/GRCh37.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/seq/GRCh37.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/seq/GRCh37.fa.fai" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/seq/GRCh37.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/seq/GRCh37.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/seq/GRCh37.fa.fai" - } - ] - } - ], - "reference__genome_context": [ - [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/bad_promoter.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/bad_promoter.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15to20.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15to20.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc20to25.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc20to25.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc25to30.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc25to30.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc65to70.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc65to70.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc70to75.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc70to75.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc75to80.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc75to80.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc80to85.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc80to85.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc85.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc85.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/self_chain.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/self_chain.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi" - } - ] - } - ], - [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/LCR.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/bad_promoter.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/bad_promoter.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15to20.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc15to20.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc20to25.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc20to25.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc25to30.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc25to30.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc65to70.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc65to70.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc70to75.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc70to75.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc75to80.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc75to80.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc80to85.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc80to85.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc85.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/gc85.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/heng_um75-hs37d5.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_51to200bp.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/low_complexity_gt200bp.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/repeats/polyx.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/self_chain.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/GA4GH/self_chain.bed.gz.tbi" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/coverage/problem_regions/ENCODE/wgEncodeDacMapabilityConsensusExcludable.bed.gz.tbi" - } - ] - } - ] - ], - "reference__rtg": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/rtg--GRCh37.sdf-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/rtg--GRCh37.sdf-wf.tar.gz" - } - ], - "reference__snpeff__GRCh37_75": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/snpeff--GRCh37.75-wf.tar.gz" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/snpeff--GRCh37.75-wf.tar.gz" - } - ], - "reference__versions": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/versions.csv" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/versions.csv" - } - ], - "reference__viral": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.amb" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.sa" - } - ] - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa", - "secondaryFiles": [ - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.dict" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.amb" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.ann" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.bwt" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.fai" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.pac" - }, - { - "class": "File", - "path": "gs://bcbiodata/collections/GRCh37/viral/gdc-viral.fa.sa" - } - ] - } - ], - "resources": [ - "{}", - "{}" - ], - "rgnames__lane": [ - "NA12878_chr21", - "NA24385_chr21" - ], - "rgnames__lb": [ - null, - null - ], - "rgnames__pl": [ - "illumina", - "illumina" - ], - "rgnames__pu": [ - "NA12878_chr21", - "NA24385_chr21" - ], - "rgnames__rg": [ - "NA12878_chr21", - "NA24385_chr21" - ], - "rgnames__sample": [ - "NA12878_chr21", - "NA24385_chr21" - ], - "vrn_file": [ - null, - null - ] -} diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test.cwl deleted file mode 100644 index c325ecade0b..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/main-wes_chr21_test.cwl +++ /dev/null @@ -1,997 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: files - secondaryFiles: - - .bai - type: - items: - items: File - type: array - type: array -- id: config__algorithm__trim_reads - type: - items: string - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: config__algorithm__vcfanno - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__svprioritize - type: - items: - - 'null' - - string - type: array -- id: resources - type: - items: string - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__adapters - type: - items: - items: string - type: array - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__gc_profile - type: - items: - - 'null' - - string - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: genome_resources__variation__lcr - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: reference__snpeff__GRCh37_75 - type: - items: File - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: reference__rtg - type: - items: File - type: array -- id: config__algorithm__validate - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: description - type: - items: string - type: array -- id: config__algorithm__validate_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: genome_resources__variation__germline_het_pon - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_off - type: - items: - items: string - type: array - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__variation__polyx - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__versions - type: - items: File - type: array -- id: genome_resources__variation__encode_blacklist - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__viral - secondaryFiles: - - .fai - - .ann - - .sa - - .pac - - .amb - - ^.dict - - .bwt - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: config__algorithm__ensemble - type: - items: string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: config__algorithm__svvalidate - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: config__algorithm__archive - type: - items: - - 'null' - - string - type: array -outputs: -- id: rgnames__sample_out - outputSource: prep_samples/rgnames__sample - type: - items: string - type: array -- id: align_bam - outputSource: postprocess_alignment/align_bam - type: - items: - - File - - 'null' - type: array -- id: regions__sample_callable - outputSource: postprocess_alignment/regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: validate__grading_summary - outputSource: summarize_vc/validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: variants__calls - outputSource: summarize_vc/variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - outputSource: summarize_vc/variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: sv__calls - outputSource: summarize_sv/sv__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: svvalidate__grading_summary - outputSource: summarize_sv/svvalidate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: sv__prioritize__tsv - outputSource: summarize_sv/sv__prioritize__tsv - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__prioritize__raw - outputSource: summarize_sv/sv__prioritize__raw - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__supplemental - outputSource: summarize_sv/sv__supplemental - type: - items: - items: - - File - type: array - type: array -- id: summary__multiqc - outputSource: multiqc_summary/summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - outputSource: multiqc_summary/versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - outputSource: multiqc_summary/versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: alignment_to_rec - in: - - id: files - source: files - - id: analysis - source: analysis - - id: config__algorithm__align_split_size - source: config__algorithm__align_split_size - - id: reference__fasta__base - source: reference__fasta__base - - id: rgnames__pl - source: rgnames__pl - - id: rgnames__sample - source: rgnames__sample - - id: rgnames__pu - source: rgnames__pu - - id: rgnames__lane - source: rgnames__lane - - id: rgnames__rg - source: rgnames__rg - - id: rgnames__lb - source: rgnames__lb - - id: reference__bwa__indexes - source: reference__bwa__indexes - - id: config__algorithm__aligner - source: config__algorithm__aligner - - id: config__algorithm__trim_reads - source: config__algorithm__trim_reads - - id: config__algorithm__adapters - source: config__algorithm__adapters - - id: config__algorithm__bam_clean - source: config__algorithm__bam_clean - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: config__algorithm__mark_duplicates - source: config__algorithm__mark_duplicates - - id: resources - source: resources - - id: description - source: description - out: - - id: alignment_rec - run: steps/alignment_to_rec.cwl -- id: alignment - in: - - id: alignment_rec - source: alignment_to_rec/alignment_rec - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: wf-alignment.cwl - scatter: - - alignment_rec - scatterMethod: dotproduct -- id: prep_samples_to_rec - in: - - id: rgnames__sample - source: rgnames__sample - - id: config__algorithm__svcaller - source: config__algorithm__svcaller - - id: config__algorithm__variant_regions - source: config__algorithm__variant_regions - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: prep_samples_rec - run: steps/prep_samples_to_rec.cwl -- id: prep_samples - in: - - id: prep_samples_rec - source: prep_samples_to_rec/prep_samples_rec - out: - - id: rgnames__sample - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - run: steps/prep_samples.cwl - scatter: - - prep_samples_rec - scatterMethod: dotproduct -- id: postprocess_alignment_to_rec - in: - - id: align_bam - source: alignment/align_bam - - id: config__algorithm__archive - source: config__algorithm__archive - - id: config__algorithm__coverage_interval - source: config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: prep_samples/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: prep_samples/config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - source: prep_samples/config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - source: prep_samples/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: prep_samples/config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - source: prep_samples/config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - source: prep_samples/config__algorithm__seq2c_bed_ready - - id: config__algorithm__recalibrate - source: config__algorithm__recalibrate - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: postprocess_alignment_rec - run: steps/postprocess_alignment_to_rec.cwl -- id: postprocess_alignment - in: - - id: postprocess_alignment_rec - source: postprocess_alignment_to_rec/postprocess_alignment_rec - out: - - id: config__algorithm__coverage_interval - - id: config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - - id: config__algorithm__variant_regions_orig - - id: config__algorithm__coverage - - id: config__algorithm__coverage_merged - - id: config__algorithm__coverage_orig - - id: config__algorithm__seq2c_bed_ready - - id: regions__callable - - id: regions__sample_callable - - id: regions__nblock - - id: depth__samtools__stats - - id: depth__samtools__idxstats - - id: depth__variant_regions__regions - - id: depth__variant_regions__dist - - id: depth__sv_regions__regions - - id: depth__sv_regions__dist - - id: depth__coverage__regions - - id: depth__coverage__dist - - id: depth__coverage__thresholds - - id: align_bam - run: steps/postprocess_alignment.cwl - scatter: - - postprocess_alignment_rec - scatterMethod: dotproduct -- id: combine_sample_regions - in: - - id: regions__callable - source: postprocess_alignment/regions__callable - - id: regions__nblock - source: postprocess_alignment/regions__nblock - - id: metadata__batch - source: metadata__batch - - id: config__algorithm__nomap_split_size - source: config__algorithm__nomap_split_size - - id: config__algorithm__nomap_split_targets - source: config__algorithm__nomap_split_targets - - id: reference__fasta__base - source: reference__fasta__base - - id: resources - source: resources - - id: description - source: description - out: - - id: config__algorithm__callable_regions - - id: config__algorithm__non_callable_regions - - id: config__algorithm__callable_count - run: steps/combine_sample_regions.cwl -- id: batch_for_variantcall - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: align_bam - source: postprocess_alignment/align_bam - - id: vrn_file - source: vrn_file - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: config__algorithm__variantcaller - source: config__algorithm__variantcaller - - id: config__algorithm__ensemble - source: config__algorithm__ensemble - - id: config__algorithm__vcfanno - source: config__algorithm__vcfanno - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__effects - source: config__algorithm__effects - - id: config__algorithm__min_allele_fraction - source: config__algorithm__min_allele_fraction - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__validate - source: config__algorithm__validate - - id: config__algorithm__validate_regions - source: config__algorithm__validate_regions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__rtg - source: reference__rtg - - id: reference__genome_context - source: reference__genome_context - - id: genome_resources__variation__clinvar - source: genome_resources__variation__clinvar - - id: genome_resources__variation__cosmic - source: genome_resources__variation__cosmic - - id: genome_resources__variation__dbsnp - source: genome_resources__variation__dbsnp - - id: genome_resources__variation__esp - source: genome_resources__variation__esp - - id: genome_resources__variation__exac - source: genome_resources__variation__exac - - id: genome_resources__variation__gnomad_exome - source: genome_resources__variation__gnomad_exome - - id: genome_resources__variation__1000g - source: genome_resources__variation__1000g - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__aliases__ensembl - source: genome_resources__aliases__ensembl - - id: genome_resources__aliases__human - source: genome_resources__aliases__human - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__GRCh37_75 - source: reference__snpeff__GRCh37_75 - - id: genome_resources__variation__train_hapmap - source: genome_resources__variation__train_hapmap - - id: genome_resources__variation__train_indels - source: genome_resources__variation__train_indels - - id: resources - source: resources - - id: description - source: description - out: - - id: batch_rec - run: steps/batch_for_variantcall.cwl -- id: variantcall - in: - - id: batch_rec - source: batch_for_variantcall/batch_rec - out: - - id: vc_rec - run: wf-variantcall.cwl - scatter: - - batch_rec - scatterMethod: dotproduct -- id: batch_for_ensemble - in: - - id: vc_rec - source: variantcall/vc_rec - out: - - id: ensemble_prep_rec - run: steps/batch_for_ensemble.cwl -- id: combine_calls - in: - - id: ensemble_prep_rec - source: batch_for_ensemble/ensemble_prep_rec - out: - - id: ensemble_rec - run: steps/combine_calls.cwl - scatter: - - ensemble_prep_rec - scatterMethod: dotproduct -- id: summarize_vc - in: - - id: vc_rec - source: variantcall/vc_rec - - id: ensemble_rec - source: combine_calls/ensemble_rec - out: - - id: variants__calls - - id: variants__gvcf - - id: variants__samples - - id: validate__grading_summary - - id: validate__grading_plots - run: steps/summarize_vc.cwl -- id: calculate_sv_bins - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: reference__fasta__base - source: reference__fasta__base - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__callable_regions - source: combine_sample_regions/config__algorithm__callable_regions - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: config__algorithm__exclude_regions - source: config__algorithm__exclude_regions - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__seq2c_bed_ready - source: postprocess_alignment/config__algorithm__seq2c_bed_ready - - id: config__algorithm__svcaller - source: config__algorithm__svcaller - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: genome_resources__variation__lcr - source: genome_resources__variation__lcr - - id: genome_resources__variation__polyx - source: genome_resources__variation__polyx - - id: genome_resources__variation__encode_blacklist - source: genome_resources__variation__encode_blacklist - - id: genome_resources__rnaseq__gene_bed - source: genome_resources__rnaseq__gene_bed - - id: resources - source: resources - - id: description - source: description - out: - - id: sv_bin_rec - run: steps/calculate_sv_bins.cwl -- id: calculate_sv_coverage - in: - - id: sv_bin_rec - source: calculate_sv_bins/sv_bin_rec - out: - - id: sv_rawcoverage_rec - run: steps/calculate_sv_coverage.cwl - scatter: - - sv_bin_rec - scatterMethod: dotproduct -- id: normalize_sv_coverage - in: - - id: sv_rawcoverage_rec - source: calculate_sv_coverage/sv_rawcoverage_rec - out: - - id: sv_coverage_rec - run: steps/normalize_sv_coverage.cwl -- id: batch_for_sv - in: - - id: analysis - source: analysis - - id: genome_build - source: genome_build - - id: work_bam_plus__disc - source: alignment/work_bam_plus__disc - - id: work_bam_plus__sr - source: alignment/work_bam_plus__sr - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: config__algorithm__svprioritize - source: config__algorithm__svprioritize - - id: config__algorithm__svvalidate - source: config__algorithm__svvalidate - - id: regions__sample_callable - source: postprocess_alignment/regions__sample_callable - - id: genome_resources__variation__gc_profile - source: genome_resources__variation__gc_profile - - id: genome_resources__variation__germline_het_pon - source: genome_resources__variation__germline_het_pon - - id: genome_resources__aliases__snpeff - source: genome_resources__aliases__snpeff - - id: reference__snpeff__GRCh37_75 - source: reference__snpeff__GRCh37_75 - - id: sv_coverage_rec - source: normalize_sv_coverage/sv_coverage_rec - - id: variants__samples - source: summarize_vc/variants__samples - out: - - id: sv_batch_rec - run: steps/batch_for_sv.cwl -- id: svcall - in: - - id: sv_batch_rec - source: batch_for_sv/sv_batch_rec - out: - - id: sv_rec - run: wf-svcall.cwl - scatter: - - sv_batch_rec - scatterMethod: dotproduct -- id: summarize_sv - in: - - id: sv_rec - source: svcall/sv_rec - out: - - id: sv__calls - - id: sv__supplemental - - id: sv__prioritize__tsv - - id: sv__prioritize__raw - - id: svvalidate__grading_summary - - id: svvalidate__grading_plots - run: steps/summarize_sv.cwl -- id: qc_to_rec - in: - - id: align_bam - source: postprocess_alignment/align_bam - - id: analysis - source: analysis - - id: reference__fasta__base - source: reference__fasta__base - - id: reference__versions - source: reference__versions - - id: config__algorithm__tools_on - source: config__algorithm__tools_on - - id: config__algorithm__tools_off - source: config__algorithm__tools_off - - id: genome_build - source: genome_build - - id: config__algorithm__qc - source: config__algorithm__qc - - id: metadata__batch - source: metadata__batch - - id: metadata__phenotype - source: metadata__phenotype - - id: config__algorithm__coverage_interval - source: postprocess_alignment/config__algorithm__coverage_interval - - id: depth__variant_regions__regions - source: postprocess_alignment/depth__variant_regions__regions - - id: depth__variant_regions__dist - source: postprocess_alignment/depth__variant_regions__dist - - id: depth__samtools__stats - source: postprocess_alignment/depth__samtools__stats - - id: depth__samtools__idxstats - source: postprocess_alignment/depth__samtools__idxstats - - id: depth__sv_regions__regions - source: postprocess_alignment/depth__sv_regions__regions - - id: depth__sv_regions__dist - source: postprocess_alignment/depth__sv_regions__dist - - id: depth__coverage__regions - source: postprocess_alignment/depth__coverage__regions - - id: depth__coverage__dist - source: postprocess_alignment/depth__coverage__dist - - id: depth__coverage__thresholds - source: postprocess_alignment/depth__coverage__thresholds - - id: config__algorithm__variant_regions - source: postprocess_alignment/config__algorithm__variant_regions - - id: config__algorithm__variant_regions_merged - source: postprocess_alignment/config__algorithm__variant_regions_merged - - id: config__algorithm__coverage - source: postprocess_alignment/config__algorithm__coverage - - id: config__algorithm__coverage_merged - source: postprocess_alignment/config__algorithm__coverage_merged - - id: variants__samples - source: summarize_vc/variants__samples - - id: reference__viral - source: reference__viral - - id: resources - source: resources - - id: description - source: description - out: - - id: qc_rec - run: steps/qc_to_rec.cwl -- id: pipeline_summary - in: - - id: qc_rec - source: qc_to_rec/qc_rec - out: - - id: qcout_rec - run: steps/pipeline_summary.cwl - scatter: - - qc_rec - scatterMethod: dotproduct -- id: multiqc_summary - in: - - id: qcout_rec - source: pipeline_summary/qcout_rec - out: - - id: summary__multiqc - - id: versions__tools - - id: versions__data - run: steps/multiqc_summary.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/alignment_to_rec.cwl deleted file mode 100644 index 506764917f5..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/alignment_to_rec.cwl +++ /dev/null @@ -1,194 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=alignment_rec:resources;description;config__algorithm__align_split_size;files;config__algorithm__trim_reads;reference__fasta__base;config__algorithm__adapters;rgnames__lb;rgnames__rg;rgnames__lane;reference__bwa__indexes;config__algorithm__bam_clean;config__algorithm__aligner;rgnames__pl;config__algorithm__mark_duplicates;analysis;rgnames__sample;config__algorithm__variant_regions;rgnames__pu -- sentinel_inputs=files:var,analysis:var,config__algorithm__align_split_size:var,reference__fasta__base:var,rgnames__pl:var,rgnames__sample:var,rgnames__pu:var,rgnames__lane:var,rgnames__rg:var,rgnames__lb:var,reference__bwa__indexes:var,config__algorithm__aligner:var,config__algorithm__trim_reads:var,config__algorithm__adapters:var,config__algorithm__bam_clean:var,config__algorithm__variant_regions:var,config__algorithm__mark_duplicates:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10418 - ramMin: 3072 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: files - secondaryFiles: - - .bai - type: - items: - items: File - type: array - type: array -- id: analysis - type: - items: string - type: array -- id: config__algorithm__align_split_size - type: - items: - - 'null' - - string - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: rgnames__pl - type: - items: string - type: array -- id: rgnames__sample - type: - items: string - type: array -- id: rgnames__pu - type: - items: string - type: array -- id: rgnames__lane - type: - items: string - type: array -- id: rgnames__rg - type: - items: string - type: array -- id: rgnames__lb - type: - items: - - 'null' - - string - type: array -- id: reference__bwa__indexes - secondaryFiles: - - ^.ann - - ^.pac - - ^.sa - - ^.bwt - type: - items: File - type: array -- id: config__algorithm__aligner - type: - items: string - type: array -- id: config__algorithm__trim_reads - type: - items: string - type: array -- id: config__algorithm__adapters - type: - items: - items: string - type: array - type: array -- id: config__algorithm__bam_clean - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: config__algorithm__mark_duplicates - type: - items: - - string - - 'null' - - boolean - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - - name: rgnames__pu - type: string - name: alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_ensemble.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_ensemble.cwl deleted file mode 100644 index df553e9e92f..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_ensemble.cwl +++ /dev/null @@ -1,273 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=ensemble_prep_rec:batch_id;variants__calls;variants__variantcallers;resources;description;batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;vrn_file;reference__fasta__base;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;metadata__batch;config__algorithm__min_allele_fraction;reference__snpeff__GRCh37_75;reference__genome_context;reference__rtg;config__algorithm__validate;config__algorithm__validate_regions;genome_build;metadata__phenotype;genome_resources__aliases__human;config__algorithm__tools_off;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=vc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_ensemble -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10240 - ramMin: 3072 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: vc_rec - type: - items: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array - type: array -outputs: -- id: ensemble_prep_rec - type: - items: - fields: - - name: batch_id - type: string - - name: variants__calls - type: - items: File - type: array - - name: variants__variantcallers - type: - items: string - type: array - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: ensemble_prep_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_sv.cwl deleted file mode 100644 index c25d67e007b..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_sv.cwl +++ /dev/null @@ -1,363 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=sv_batch_rec:resources;description;config__algorithm__svprioritize;genome_resources__variation__gc_profile;reference__snpeff__GRCh37_75;genome_build;genome_resources__variation__germline_het_pon;config__algorithm__tools_off;analysis;config__algorithm__tools_on;config__algorithm__svvalidate;genome_resources__aliases__snpeff;work_bam_plus__disc;work_bam_plus__sr;regions__sample_callable;variants__samples;depth__bins__normalized;depth__bins__background;depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,work_bam_plus__disc:var,work_bam_plus__sr:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,config__algorithm__svprioritize:var,config__algorithm__svvalidate:var,regions__sample_callable:var,genome_resources__variation__gc_profile:var,genome_resources__variation__germline_het_pon:var,genome_resources__aliases__snpeff:var,reference__snpeff__GRCh37_75:var,sv_coverage_rec:record,variants__samples:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10240 - ramMin: 49152 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 3656 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - items: string - type: array - type: array -- id: config__algorithm__svprioritize - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__svvalidate - type: - items: - - 'null' - - string - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: genome_resources__variation__gc_profile - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__germline_het_pon - type: - items: - - 'null' - - string - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__GRCh37_75 - type: - items: File - type: array -- id: sv_coverage_rec - type: - items: - fields: - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_coverage_rec - type: record - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -outputs: -- id: sv_batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: - - 'null' - - string - - name: reference__snpeff__GRCh37_75 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_variantcall.cwl deleted file mode 100644 index 6504ed26811..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/batch_for_variantcall.cwl +++ /dev/null @@ -1,395 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-batch -- sentinel_outputs=batch_rec:resources;description;reference__fasta__base;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;genome_resources__variation__clinvar;genome_resources__variation__esp;metadata__batch;genome_resources__variation__lcr;config__algorithm__min_allele_fraction;reference__snpeff__GRCh37_75;vrn_file;genome_resources__variation__train_hapmap;reference__genome_context;reference__rtg;config__algorithm__validate;genome_resources__variation__1000g;config__algorithm__validate_regions;genome_build;genome_resources__variation__exac;genome_resources__variation__gnomad_exome;metadata__phenotype;genome_resources__aliases__human;config__algorithm__tools_off;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;genome_resources__variation__cosmic;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;genome_resources__variation__train_indels;genome_resources__aliases__snpeff;align_bam;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=analysis:var,genome_build:var,align_bam:var,vrn_file:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,regions__sample_callable:var,config__algorithm__variantcaller:var,config__algorithm__ensemble:var,config__algorithm__vcfanno:var,config__algorithm__coverage_interval:var,config__algorithm__effects:var,config__algorithm__min_allele_fraction:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__validate:var,config__algorithm__validate_regions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,reference__fasta__base:var,reference__rtg:var,reference__genome_context:var,genome_resources__variation__clinvar:var,genome_resources__variation__cosmic:var,genome_resources__variation__dbsnp:var,genome_resources__variation__esp:var,genome_resources__variation__exac:var,genome_resources__variation__gnomad_exome:var,genome_resources__variation__1000g:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__aliases__ensembl:var,genome_resources__aliases__human:var,genome_resources__aliases__snpeff:var,reference__snpeff__GRCh37_75:var,genome_resources__variation__train_hapmap:var,genome_resources__variation__train_indels:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- batch_for_variantcall -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10477 - ramMin: 3072 - tmpdirMin: 119 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: analysis - type: - items: string - type: array -- id: genome_build - type: - items: string - type: array -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: vrn_file - type: - items: - - 'null' - - string - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: regions__sample_callable - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variantcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__ensemble - type: - items: string - type: array -- id: config__algorithm__vcfanno - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__effects - type: - items: string - type: array -- id: config__algorithm__min_allele_fraction - type: - items: double - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__validate - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__validate_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - items: string - type: array - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: reference__rtg - type: - items: File - type: array -- id: reference__genome_context - secondaryFiles: - - .tbi - type: - items: - items: File - type: array - type: array -- id: genome_resources__variation__clinvar - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__cosmic - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__esp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__exac - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__gnomad_exome - type: - items: - - 'null' - - string - type: array -- id: genome_resources__variation__1000g - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__polyx - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__encode_blacklist - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__aliases__ensembl - type: - items: string - type: array -- id: genome_resources__aliases__human - type: - items: - - string - - 'null' - - boolean - type: array -- id: genome_resources__aliases__snpeff - type: - items: string - type: array -- id: reference__snpeff__GRCh37_75 - type: - items: File - type: array -- id: genome_resources__variation__train_hapmap - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__train_indels - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: batch_rec - type: - items: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/calculate_sv_bins.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/calculate_sv_bins.cwl deleted file mode 100644 index bee219164db..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/calculate_sv_bins.cwl +++ /dev/null @@ -1,219 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv_bin_rec:regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;resources;description;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=align_bam:var,reference__fasta__base:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__callable_regions:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__seq2c_bed_ready:var,config__algorithm__svcaller:var,depth__variant_regions__regions:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,genome_resources__rnaseq__gene_bed:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- calculate_sv_bins -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10418 - ramMin: 3072 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 3024 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__seq2c_bed_ready - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: genome_resources__variation__lcr - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__polyx - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__encode_blacklist - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: sv_bin_rec - type: - items: - fields: - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_bin_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/calculate_sv_coverage.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/calculate_sv_coverage.cwl deleted file mode 100644 index 330224b83db..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/calculate_sv_coverage.cwl +++ /dev/null @@ -1,206 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=sv_rawcoverage_rec:depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;resources;description;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_bin_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- calculate_sv_coverage -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10418 - ramMin: 49152 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 3024 -- class: SoftwareRequirement - packages: - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit - - package: seq2c - specs: - - https://anaconda.org/bioconda/seq2c -inputs: -- id: sv_bin_rec - type: - fields: - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_bin_rec - type: record -outputs: -- id: sv_rawcoverage_rec - type: - fields: - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rawcoverage_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/combine_calls.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/combine_calls.cwl deleted file mode 100644 index 7730a2a3223..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/combine_calls.cwl +++ /dev/null @@ -1,180 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=ensemble_rec:ensemble__vrn_file;ensemble__validate__summary;ensemble__batch_samples;ensemble__batch_id -- sentinel_inputs=ensemble_prep_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- combine_calls -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10240 - ramMin: 49152 - tmpdirMin: 0 -- class: dx:InputResourceRequirement - indirMin: 4687 -- class: SoftwareRequirement - packages: - - package: bcbio-variation-recall - specs: - - https://anaconda.org/bioconda/bcbio-variation-recall -inputs: -- id: ensemble_prep_rec - type: - fields: - - name: batch_id - type: string - - name: variants__calls - type: - items: File - type: array - - name: variants__variantcallers - type: - items: string - type: array - - name: resources - type: string - - name: description - type: string - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: ensemble_prep_rec - type: record -outputs: -- id: ensemble_rec - type: - fields: - - name: ensemble__vrn_file - type: - - File - - 'null' - - name: ensemble__validate__summary - type: - - File - - 'null' - - name: ensemble__batch_samples - type: - items: string - type: array - - name: ensemble__batch_id - type: string - name: ensemble_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/combine_sample_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/combine_sample_regions.cwl deleted file mode 100644 index e59fc89683e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/combine_sample_regions.cwl +++ /dev/null @@ -1,99 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=config__algorithm__callable_regions,config__algorithm__non_callable_regions,config__algorithm__callable_count -- sentinel_inputs=regions__callable:var,regions__nblock:var,metadata__batch:var,config__algorithm__nomap_split_size:var,config__algorithm__nomap_split_targets:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- combine_sample_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10299 - ramMin: 3072 - tmpdirMin: 30 -- class: dx:InputResourceRequirement - indirMin: 3008 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: regions__callable - type: - items: - - File - - 'null' - type: array -- id: regions__nblock - type: - items: - - File - - 'null' - type: array -- id: metadata__batch - type: - items: string - type: array -- id: config__algorithm__nomap_split_size - type: - items: long - type: array -- id: config__algorithm__nomap_split_targets - type: - items: long - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: config__algorithm__callable_regions - type: - items: File - type: array -- id: config__algorithm__non_callable_regions - type: - items: File - type: array -- id: config__algorithm__callable_count - type: - items: int - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/compare_to_rm.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/compare_to_rm.cwl deleted file mode 100644 index ed7c9b1e0b1..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/compare_to_rm.cwl +++ /dev/null @@ -1,299 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vc_rec:batch_samples;validate__summary;validate__tp;validate__fp;validate__fn;resources;description;vrn_file;reference__fasta__base;config__algorithm__vcfanno;config__algorithm__variantcaller;config__algorithm__coverage_interval;metadata__batch;config__algorithm__min_allele_fraction;reference__snpeff__GRCh37_75;reference__genome_context;reference__rtg;config__algorithm__validate;config__algorithm__validate_regions;genome_build;metadata__phenotype;genome_resources__aliases__human;config__algorithm__tools_off;config__algorithm__ensemble;analysis;config__algorithm__tools_on;config__algorithm__effects;config__algorithm__variant_regions;genome_resources__aliases__ensembl;config__algorithm__exclude_regions;genome_resources__aliases__snpeff;config__algorithm__variant_regions_merged;regions__sample_callable;config__algorithm__callable_regions -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- compare_to_rm -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10418 - ramMin: 49152 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 24883 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: gvcf-regions - specs: - - https://anaconda.org/bioconda/gvcf-regions - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: rtg-tools - specs: - - https://anaconda.org/bioconda/rtg-tools - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file - secondaryFiles: - - .tbi - type: File -outputs: -- id: vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/concat_batch_variantcalls.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/concat_batch_variantcalls.cwl deleted file mode 100644 index a5d59d661a0..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/concat_batch_variantcalls.cwl +++ /dev/null @@ -1,190 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-merge -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,region_block:var,vrn_file_region:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- concat_batch_variantcalls -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10418 - ramMin: 3072 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 24883 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 -- class: arv:APIRequirement -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block - type: - items: - items: string - type: array - type: array -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - items: - - File - - 'null' - type: array -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/detect_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/detect_sv.cwl deleted file mode 100644 index 66d592642d7..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/detect_sv.cwl +++ /dev/null @@ -1,402 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=sv_rec:sv__variantcaller;sv__vrn_file;sv__supplemental;svvalidate__summary;resources;description;config__algorithm__svprioritize;genome_resources__variation__gc_profile;genome_build;genome_resources__variation__germline_het_pon;config__algorithm__tools_off;analysis;config__algorithm__tools_on;config__algorithm__svvalidate;genome_resources__aliases__snpeff;regions__sample_callable;variants__samples;depth__bins__normalized;depth__bins__background;depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- detect_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10477 - ramMin: 49152 - tmpdirMin: 119 -- class: dx:InputResourceRequirement - indirMin: 3656 -- class: SoftwareRequirement - packages: - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit - - package: delly - specs: - - https://anaconda.org/bioconda/delly - - package: duphold - specs: - - https://anaconda.org/bioconda/duphold - - package: extract-sv-reads - specs: - - https://anaconda.org/bioconda/extract-sv-reads - - package: lumpy-sv - specs: - - https://anaconda.org/bioconda/lumpy-sv - - package: manta - specs: - - https://anaconda.org/bioconda/manta - - package: break-point-inspector - specs: - - https://anaconda.org/bioconda/break-point-inspector - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: smoove - specs: - - https://anaconda.org/bioconda/smoove - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: seq2c - specs: - - https://anaconda.org/bioconda/seq2c - - package: simple_sv_annotation - specs: - - https://anaconda.org/bioconda/simple_sv_annotation - - package: survivor - specs: - - https://anaconda.org/bioconda/survivor - - package: svtools - specs: - - https://anaconda.org/bioconda/svtools - - package: svtyper - specs: - - https://anaconda.org/bioconda/svtyper - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: xorg-libxt - specs: - - https://anaconda.org/bioconda/xorg-libxt - - package: vawk - specs: - - https://anaconda.org/bioconda/vawk -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: sv_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: - - 'null' - - string - - name: reference__snpeff__GRCh37_75 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array -outputs: -- id: sv_rec - type: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/get_parallel_regions.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/get_parallel_regions.cwl deleted file mode 100644 index 71c08911610..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/get_parallel_regions.cwl +++ /dev/null @@ -1,165 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-split -- sentinel_outputs=region_block -- sentinel_inputs=batch_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- get_parallel_regions -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10477 - ramMin: 3072 - tmpdirMin: 119 -- class: dx:InputResourceRequirement - indirMin: 24883 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: region_block - type: - items: - items: string - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/merge_split_alignments.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/merge_split_alignments.cwl deleted file mode 100644 index 4e139a4d748..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/merge_split_alignments.cwl +++ /dev/null @@ -1,165 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-merge -- sentinel_outputs=align_bam,work_bam_plus__disc,work_bam_plus__sr,hla__fastq -- sentinel_inputs=alignment_rec:record,work_bam:var,align_bam:var,work_bam_plus__disc:var,work_bam_plus__sr:var,hla__fastq:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- merge_split_alignments -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10655 - ramMin: 49152 - tmpdirMin: 208 -- class: dx:InputResourceRequirement - indirMin: 3127 -- class: SoftwareRequirement - packages: - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - - name: rgnames__pu - type: string - name: alignment_rec - type: record -- id: work_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: align_bam_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__disc_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: work_bam_plus__sr_toolinput - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: hla__fastq_toolinput - type: - items: - - 'null' - - items: File - type: array - type: array -outputs: -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/multiqc_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/multiqc_summary.cwl deleted file mode 100644 index 96036a07fc2..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/multiqc_summary.cwl +++ /dev/null @@ -1,94 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=summary__multiqc,versions__tools,versions__data -- sentinel_inputs=qcout_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- multiqc_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10477 - ramMin: 3072 - tmpdirMin: 119 -- class: dx:InputResourceRequirement - indirMin: 1 -- class: SoftwareRequirement - packages: - - package: multiqc - specs: - - https://anaconda.org/bioconda/multiqc - - package: multiqc-bcbio - specs: - - https://anaconda.org/bioconda/multiqc-bcbio -inputs: -- id: qcout_rec - type: - items: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: reference__versions - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record - type: array -outputs: -- id: summary__multiqc - type: - items: - - File - - 'null' - type: array -- id: versions__tools - type: - items: - - File - - 'null' - type: array -- id: versions__data - type: - items: - - File - - 'null' - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/normalize_sv_coverage.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/normalize_sv_coverage.cwl deleted file mode 100644 index af33573637d..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/normalize_sv_coverage.cwl +++ /dev/null @@ -1,224 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv_coverage_rec:depth__bins__normalized;depth__bins__background;resources;description;depth__bins__target;depth__bins__antitarget;depth__bins__seq2c;regions__bins__target;regions__bins__antitarget;regions__bins__gcannotated;regions__bins__group;reference__fasta__base;config__algorithm__svcaller;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;metadata__batch;genome_resources__variation__lcr;metadata__phenotype;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__variant_regions;config__algorithm__exclude_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__seq2c_bed_ready;depth__variant_regions__regions;config__algorithm__callable_regions -- sentinel_inputs=sv_rawcoverage_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- normalize_sv_coverage -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10418 - ramMin: 49152 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 3024 -- class: SoftwareRequirement - packages: - - package: cnvkit - specs: - - https://anaconda.org/bioconda/cnvkit -inputs: -- id: sv_rawcoverage_rec - type: - items: - fields: - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rawcoverage_rec - type: record - type: array -outputs: -- id: sv_coverage_rec - type: - items: - fields: - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_coverage_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/pipeline_summary.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/pipeline_summary.cwl deleted file mode 100644 index 4ffe2646ec9..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/pipeline_summary.cwl +++ /dev/null @@ -1,219 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=qcout_rec:summary__qc;summary__metrics;description;genome_build;config__algorithm__tools_off;reference__versions;config__algorithm__qc;config__algorithm__tools_on -- sentinel_inputs=qc_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- pipeline_summary -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10477 - ramMin: 49152 - tmpdirMin: 119 -- class: dx:InputResourceRequirement - indirMin: 3010 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: fastqc=0.11.7=5 - specs: - - https://anaconda.org/bioconda/fastqc=0.11.7=5 - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: hts-nim-tools - specs: - - https://anaconda.org/bioconda/hts-nim-tools - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: qsignature - specs: - - https://anaconda.org/bioconda/qsignature - - package: qualimap - specs: - - https://anaconda.org/bioconda/qualimap - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: preseq - specs: - - https://anaconda.org/bioconda/preseq - - package: peddy - specs: - - https://anaconda.org/bioconda/peddy - - package: verifybamid2 - specs: - - https://anaconda.org/bioconda/verifybamid2 -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: qc_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: reference__versions - type: File - - name: reference__viral - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record -outputs: -- id: qcout_rec - type: - fields: - - name: summary__qc - type: - - File - - 'null' - - name: summary__metrics - type: - - string - - 'null' - - name: description - type: string - - name: genome_build - type: string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: reference__versions - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: config__algorithm__tools_on - type: - items: string - type: array - name: qcout_rec - type: record -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_alignment.cwl deleted file mode 100644 index 92f4acb8c7f..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_alignment.cwl +++ /dev/null @@ -1,221 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=config__algorithm__coverage_interval,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready,regions__callable,regions__sample_callable,regions__nblock,depth__samtools__stats,depth__samtools__idxstats,depth__variant_regions__regions,depth__variant_regions__dist,depth__sv_regions__regions,depth__sv_regions__dist,depth__coverage__regions,depth__coverage__dist,depth__coverage__thresholds,align_bam -- sentinel_inputs=postprocess_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10596 - ramMin: 49152 - tmpdirMin: 178 -- class: dx:InputResourceRequirement - indirMin: 18018 -- class: SoftwareRequirement - packages: - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: goleft - specs: - - https://anaconda.org/bioconda/goleft - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: mosdepth - specs: - - https://anaconda.org/bioconda/mosdepth - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon -- class: arv:APIRequirement -inputs: -- id: postprocess_alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__archive - type: - - 'null' - - string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record -outputs: -- id: config__algorithm__coverage_interval - type: - - string - - 'null' -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -- id: regions__callable - type: - - File - - 'null' -- id: regions__sample_callable - type: - - File - - 'null' -- id: regions__nblock - type: - - File - - 'null' -- id: depth__samtools__stats - type: - - File - - 'null' -- id: depth__samtools__idxstats - type: - - File - - 'null' -- id: depth__variant_regions__regions - type: - - File - - 'null' -- id: depth__variant_regions__dist - type: - - File - - 'null' -- id: depth__sv_regions__regions - type: - - File - - 'null' -- id: depth__sv_regions__dist - type: - - File - - 'null' -- id: depth__coverage__regions - type: - - File - - 'null' -- id: depth__coverage__dist - type: - - File - - 'null' -- id: depth__coverage__thresholds - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_alignment_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_alignment_to_rec.cwl deleted file mode 100644 index 926d37e91df..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_alignment_to_rec.cwl +++ /dev/null @@ -1,237 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=postprocess_alignment_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;genome_resources__rnaseq__gene_bed;genome_resources__variation__lcr;config__algorithm__recalibrate;genome_resources__variation__dbsnp;genome_resources__variation__polyx;genome_resources__variation__encode_blacklist;config__algorithm__tools_on;config__algorithm__variant_regions;config__algorithm__exclude_regions;config__algorithm__archive;align_bam;config__algorithm__variant_regions_merged;config__algorithm__variant_regions_orig;config__algorithm__coverage;config__algorithm__coverage_merged;config__algorithm__coverage_orig;config__algorithm__seq2c_bed_ready -- sentinel_inputs=align_bam:var,config__algorithm__archive:var,config__algorithm__coverage_interval:var,config__algorithm__exclude_regions:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__variant_regions_orig:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,config__algorithm__coverage_orig:var,config__algorithm__seq2c_bed_ready:var,config__algorithm__recalibrate:var,config__algorithm__tools_on:var,genome_resources__rnaseq__gene_bed:var,genome_resources__variation__dbsnp:var,genome_resources__variation__lcr:var,genome_resources__variation__polyx:var,genome_resources__variation__encode_blacklist:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_alignment_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10418 - ramMin: 3072 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__archive - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - 'null' - - string - type: array -- id: config__algorithm__exclude_regions - type: - items: - - 'null' - - items: 'null' - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_orig - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__seq2c_bed_ready - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__recalibrate - type: - items: - - string - - 'null' - - boolean - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: genome_resources__rnaseq__gene_bed - type: - items: File - type: array -- id: genome_resources__variation__dbsnp - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__lcr - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__polyx - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: genome_resources__variation__encode_blacklist - secondaryFiles: - - .tbi - type: - items: File - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: postprocess_alignment_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - 'null' - - string - - name: genome_resources__rnaseq__gene_bed - type: File - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__recalibrate - type: - - string - - 'null' - - boolean - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__archive - type: - - 'null' - - string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__variant_regions_orig - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: config__algorithm__coverage_orig - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - name: postprocess_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_variants.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_variants.cwl deleted file mode 100644 index 5151f1520c6..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/postprocess_variants.cwl +++ /dev/null @@ -1,174 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-single -- sentinel_outputs=vrn_file -- sentinel_inputs=batch_rec:record,vrn_file:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- postprocess_variants -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10299 - ramMin: 49152 - tmpdirMin: 30 -- class: dx:InputResourceRequirement - indirMin: 24883 -- class: SoftwareRequirement - packages: - - package: snpeff - specs: - - https://anaconda.org/bioconda/snpeff - version: - - 4.3.1t -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: vrn_file_toolinput - secondaryFiles: - - .tbi - type: File -outputs: -- id: vrn_file - secondaryFiles: - - .tbi - type: File -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_align_inputs.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_align_inputs.cwl deleted file mode 100644 index 5c392bdb350..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_align_inputs.cwl +++ /dev/null @@ -1,139 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-split -- sentinel_outputs=process_alignment_rec:files;config__algorithm__quality_format;align_split -- sentinel_inputs=alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_align_inputs -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10418 - ramMin: 49152 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 3127 -- class: SoftwareRequirement - packages: - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: atropos;env - specs: - - https://anaconda.org/bioconda/atropos;env - version: - - python3 - - package: optitype - specs: - - https://anaconda.org/bioconda/optitype - - package: razers3 - specs: - - https://anaconda.org/bioconda/razers3 - version: - - 3.5.0 - - package: coincbc - specs: - - https://anaconda.org/bioconda/coincbc -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - - name: rgnames__pu - type: string - name: alignment_rec - type: record -outputs: -- id: process_alignment_rec - type: - items: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_samples.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_samples.cwl deleted file mode 100644 index 8e6d9500406..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_samples.cwl +++ /dev/null @@ -1,95 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-parallel -- sentinel_outputs=rgnames__sample,config__algorithm__variant_regions,config__algorithm__variant_regions_merged,config__algorithm__variant_regions_orig,config__algorithm__coverage,config__algorithm__coverage_merged,config__algorithm__coverage_orig,config__algorithm__seq2c_bed_ready -- sentinel_inputs=prep_samples_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10299 - ramMin: 3072 - tmpdirMin: 30 -- class: dx:InputResourceRequirement - indirMin: 3008 -- class: SoftwareRequirement - packages: - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy -inputs: -- id: prep_samples_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record -outputs: -- id: rgnames__sample - type: string -- id: config__algorithm__variant_regions - type: - - File - - 'null' -- id: config__algorithm__variant_regions_merged - type: - - File - - 'null' -- id: config__algorithm__variant_regions_orig - type: - - File - - 'null' -- id: config__algorithm__coverage - type: - - File - - 'null' -- id: config__algorithm__coverage_merged - type: - - File - - 'null' -- id: config__algorithm__coverage_orig - type: - - File - - 'null' -- id: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_samples_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_samples_to_rec.cwl deleted file mode 100644 index 9ccaa591fde..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/prep_samples_to_rec.cwl +++ /dev/null @@ -1,85 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=prep_samples_rec:resources;description;reference__fasta__base;config__algorithm__svcaller;rgnames__sample;config__algorithm__variant_regions -- sentinel_inputs=rgnames__sample:var,config__algorithm__svcaller:var,config__algorithm__variant_regions:var,reference__fasta__base:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- prep_samples_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10299 - ramMin: 3072 - tmpdirMin: 30 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: rgnames__sample - type: - items: string - type: array -- id: config__algorithm__svcaller - type: - items: - items: string - type: array - type: array -- id: config__algorithm__variant_regions - type: - items: File - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: prep_samples_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: - items: string - type: array - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - name: prep_samples_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/process_alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/process_alignment.cwl deleted file mode 100644 index 718e2cd14e0..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/process_alignment.cwl +++ /dev/null @@ -1,195 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=single-parallel -- sentinel_outputs=work_bam,align_bam,hla__fastq,work_bam_plus__disc,work_bam_plus__sr -- sentinel_inputs=alignment_rec:record,process_alignment_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- process_alignment -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10477 - ramMin: 49152 - tmpdirMin: 119 -- class: dx:InputResourceRequirement - indirMin: 3246 -- class: SoftwareRequirement - packages: - - package: bwa - specs: - - https://anaconda.org/bioconda/bwa - - package: bwakit - specs: - - https://anaconda.org/bioconda/bwakit - - package: grabix - specs: - - https://anaconda.org/bioconda/grabix - - package: minimap2 - specs: - - https://anaconda.org/bioconda/minimap2 - - package: novoalign - specs: - - https://anaconda.org/bioconda/novoalign - - package: snap-aligner - specs: - - https://anaconda.org/bioconda/snap-aligner - version: - - 1.0dev.97 - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: sambamba - specs: - - https://anaconda.org/bioconda/sambamba - - package: fgbio - specs: - - https://anaconda.org/bioconda/fgbio - - package: umis - specs: - - https://anaconda.org/bioconda/umis - - package: biobambam - specs: - - https://anaconda.org/bioconda/biobambam - - package: seqtk - specs: - - https://anaconda.org/bioconda/seqtk - - package: samblaster - specs: - - https://anaconda.org/bioconda/samblaster - - package: variantbam - specs: - - https://anaconda.org/bioconda/variantbam -- class: arv:APIRequirement -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - - name: rgnames__pu - type: string - name: alignment_rec - type: record -- id: process_alignment_rec - type: - fields: - - name: files - type: - - 'null' - - items: File - type: array - - name: config__algorithm__quality_format - type: - - string - - 'null' - - name: align_split - type: - - string - - 'null' - name: process_alignment_rec - type: record -outputs: -- id: work_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - type: - - 'null' - - items: File - type: array -- id: work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/qc_to_rec.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/qc_to_rec.cwl deleted file mode 100644 index 0336669ad83..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/qc_to_rec.cwl +++ /dev/null @@ -1,307 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=qc_rec:resources;description;reference__fasta__base;config__algorithm__coverage_interval;metadata__batch;genome_build;metadata__phenotype;config__algorithm__tools_off;reference__versions;reference__viral;config__algorithm__qc;analysis;config__algorithm__tools_on;config__algorithm__variant_regions;align_bam;config__algorithm__variant_regions_merged;config__algorithm__coverage;config__algorithm__coverage_merged;depth__samtools__stats;depth__samtools__idxstats;depth__variant_regions__regions;depth__variant_regions__dist;depth__sv_regions__regions;depth__sv_regions__dist;depth__coverage__regions;depth__coverage__dist;depth__coverage__thresholds;variants__samples -- sentinel_inputs=align_bam:var,analysis:var,reference__fasta__base:var,reference__versions:var,config__algorithm__tools_on:var,config__algorithm__tools_off:var,genome_build:var,config__algorithm__qc:var,metadata__batch:var,metadata__phenotype:var,config__algorithm__coverage_interval:var,depth__variant_regions__regions:var,depth__variant_regions__dist:var,depth__samtools__stats:var,depth__samtools__idxstats:var,depth__sv_regions__regions:var,depth__sv_regions__dist:var,depth__coverage__regions:var,depth__coverage__dist:var,depth__coverage__thresholds:var,config__algorithm__variant_regions:var,config__algorithm__variant_regions_merged:var,config__algorithm__coverage:var,config__algorithm__coverage_merged:var,variants__samples:var,reference__viral:var,resources:var,description:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- qc_to_rec -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10418 - ramMin: 3072 - tmpdirMin: 89 -- class: dx:InputResourceRequirement - indirMin: 0 -inputs: -- id: align_bam - secondaryFiles: - - .bai - type: - items: - - File - - 'null' - type: array -- id: analysis - type: - items: string - type: array -- id: reference__fasta__base - secondaryFiles: - - ^.dict - - .fai - type: - items: File - type: array -- id: reference__versions - type: - items: File - type: array -- id: config__algorithm__tools_on - type: - items: - items: string - type: array - type: array -- id: config__algorithm__tools_off - type: - items: - items: string - type: array - type: array -- id: genome_build - type: - items: string - type: array -- id: config__algorithm__qc - type: - items: - items: string - type: array - type: array -- id: metadata__batch - type: - items: string - type: array -- id: metadata__phenotype - type: - items: string - type: array -- id: config__algorithm__coverage_interval - type: - items: - - string - - 'null' - type: array -- id: depth__variant_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__variant_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__stats - type: - items: - - File - - 'null' - type: array -- id: depth__samtools__idxstats - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__regions - type: - items: - - File - - 'null' - type: array -- id: depth__sv_regions__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__regions - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__dist - type: - items: - - File - - 'null' - type: array -- id: depth__coverage__thresholds - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__variant_regions_merged - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage - type: - items: - - File - - 'null' - type: array -- id: config__algorithm__coverage_merged - type: - items: - - File - - 'null' - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: reference__viral - secondaryFiles: - - .fai - - .ann - - .sa - - .pac - - .amb - - ^.dict - - .bwt - type: - items: File - type: array -- id: resources - type: - items: string - type: array -- id: description - type: - items: string - type: array -outputs: -- id: qc_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: reference__versions - type: File - - name: reference__viral - type: File - - name: config__algorithm__qc - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__coverage - type: - - File - - 'null' - - name: config__algorithm__coverage_merged - type: - - File - - 'null' - - name: depth__samtools__stats - type: - - File - - 'null' - - name: depth__samtools__idxstats - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: depth__variant_regions__dist - type: - - File - - 'null' - - name: depth__sv_regions__regions - type: - - File - - 'null' - - name: depth__sv_regions__dist - type: - - File - - 'null' - - name: depth__coverage__regions - type: - - File - - 'null' - - name: depth__coverage__dist - type: - - File - - 'null' - - name: depth__coverage__thresholds - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - name: qc_rec - type: record - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/summarize_sv.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/summarize_sv.cwl deleted file mode 100644 index c5f8047d69a..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/summarize_sv.cwl +++ /dev/null @@ -1,236 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=sv__calls,sv__supplemental,sv__prioritize__tsv,sv__prioritize__raw,svvalidate__grading_summary,svvalidate__grading_plots -- sentinel_inputs=sv_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_sv -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 10358 - ramMin: 3072 - tmpdirMin: 59 -- class: dx:InputResourceRequirement - indirMin: 3024 -- class: SoftwareRequirement - packages: - - package: bcbio-prioritize - specs: - - https://anaconda.org/bioconda/bcbio-prioritize -inputs: -- id: sv_rec - type: - items: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array - type: array -outputs: -- id: sv__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__supplemental - type: - items: - items: - - File - type: array - type: array -- id: sv__prioritize__tsv - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: sv__prioritize__raw - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: svvalidate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: svvalidate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/summarize_vc.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/summarize_vc.cwl deleted file mode 100644 index 48ec1cd601a..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/summarize_vc.cwl +++ /dev/null @@ -1,212 +0,0 @@ -$namespaces: - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=multi-combined -- sentinel_outputs=variants__calls,variants__gvcf,variants__samples,validate__grading_summary,validate__grading_plots -- sentinel_inputs=vc_rec:record,ensemble_rec:record -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- summarize_vc -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 1 - outdirMin: 1000000 - ramMin: 3072 - tmpdirMin: 1000000 -- class: dx:InputResourceRequirement - indirMin: 1000000 -inputs: -- id: vc_rec - type: - items: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array - type: array -- id: ensemble_rec - type: - items: - fields: - - name: ensemble__vrn_file - type: - - File - - 'null' - - name: ensemble__validate__summary - type: - - File - - 'null' - - name: ensemble__batch_samples - type: - items: string - type: array - - name: ensemble__batch_id - type: string - name: ensemble_rec - type: record - type: array -outputs: -- id: variants__calls - type: - items: - items: - - File - - 'null' - type: array - type: array -- id: variants__gvcf - type: - items: - - 'null' - - items: - - File - - 'null' - type: array - type: array -- id: variants__samples - type: - items: - items: - items: - - File - - 'null' - type: array - type: array - type: array -- id: validate__grading_summary - type: - items: - - File - - 'null' - type: array -- id: validate__grading_plots - type: - items: - items: - - File - - 'null' - type: array - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/variantcall_batch_region.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/variantcall_batch_region.cwl deleted file mode 100644 index a4fadb20991..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/steps/variantcall_batch_region.cwl +++ /dev/null @@ -1,260 +0,0 @@ -$namespaces: - arv: http://arvados.org/cwl# - dx: https://www.dnanexus.com/cwl# -arguments: -- position: 0 - valueFrom: sentinel_runtime=cores,$(runtime['cores']),ram,$(runtime['ram']) -- sentinel_parallel=batch-parallel -- sentinel_outputs=vrn_file_region,region_block -- sentinel_inputs=batch_rec:record,region_block:var -- run_number=0 -baseCommand: -- bcbio_nextgen.py -- runfn -- variantcall_batch_region -- cwl -class: CommandLineTool -cwlVersion: v1.0 -hints: -- class: DockerRequirement - dockerImageId: quay.io/bcbio/bcbio-vc:1.1.4a-741877e - dockerPull: quay.io/bcbio/bcbio-vc:1.1.4a-741877e -- class: ResourceRequirement - coresMin: 16 - outdirMin: 10477 - ramMin: 49152 - tmpdirMin: 119 -- class: dx:InputResourceRequirement - indirMin: 24883 -- class: SoftwareRequirement - packages: - - package: bcftools - specs: - - https://anaconda.org/bioconda/bcftools - - package: bedtools - specs: - - https://anaconda.org/bioconda/bedtools - - package: freebayes - specs: - - https://anaconda.org/bioconda/freebayes - version: - - 1.1.0.46 - - package: gatk4 - specs: - - https://anaconda.org/bioconda/gatk4 - - package: vqsr_cnn - specs: - - https://anaconda.org/bioconda/vqsr_cnn - - package: deepvariant - specs: - - https://anaconda.org/bioconda/deepvariant - - package: sentieon - specs: - - https://anaconda.org/bioconda/sentieon - - package: htslib - specs: - - https://anaconda.org/bioconda/htslib - - package: octopus - specs: - - https://anaconda.org/bioconda/octopus - - package: picard - specs: - - https://anaconda.org/bioconda/picard - - package: platypus-variant - specs: - - https://anaconda.org/bioconda/platypus-variant - - package: pythonpy - specs: - - https://anaconda.org/bioconda/pythonpy - - package: samtools - specs: - - https://anaconda.org/bioconda/samtools - - package: pysam> - specs: - - https://anaconda.org/bioconda/pysam> - version: - - 0.13.0 - - package: strelka - specs: - - https://anaconda.org/bioconda/strelka - - package: vardict - specs: - - https://anaconda.org/bioconda/vardict - - package: vardict-java - specs: - - https://anaconda.org/bioconda/vardict-java - - package: varscan - specs: - - https://anaconda.org/bioconda/varscan - - package: moreutils - specs: - - https://anaconda.org/bioconda/moreutils - - package: vcfanno - specs: - - https://anaconda.org/bioconda/vcfanno - - package: vcflib - specs: - - https://anaconda.org/bioconda/vcflib - - package: vt - specs: - - https://anaconda.org/bioconda/vt - - package: r - specs: - - https://anaconda.org/bioconda/r - version: - - 3.4.1 - - package: r-base=3.4.1=h4fe35fd_8 - specs: - - https://anaconda.org/bioconda/r-base=3.4.1=h4fe35fd_8 - - package: perl - specs: - - https://anaconda.org/bioconda/perl -- class: arv:APIRequirement -- class: arv:RuntimeConstraints - keep_cache: 4096 -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -- id: region_block_toolinput - type: - items: string - type: array -outputs: -- id: vrn_file_region - secondaryFiles: - - .tbi - type: - - File - - 'null' -- id: region_block - type: - items: string - type: array -requirements: -- class: InlineJavascriptRequirement -- class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-alignment.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-alignment.cwl deleted file mode 100644 index 0f1dfd3dd81..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-alignment.cwl +++ /dev/null @@ -1,140 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: alignment_rec - type: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__align_split_size - type: - - 'null' - - string - - name: files - type: - items: File - type: array - - name: config__algorithm__trim_reads - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__adapters - type: - items: string - type: array - - name: rgnames__lb - type: - - 'null' - - string - - name: rgnames__rg - type: string - - name: rgnames__lane - type: string - - name: reference__bwa__indexes - type: File - - name: config__algorithm__bam_clean - type: - - string - - 'null' - - boolean - - name: config__algorithm__aligner - type: string - - name: rgnames__pl - type: string - - name: config__algorithm__mark_duplicates - type: - - string - - 'null' - - boolean - - name: analysis - type: string - - name: rgnames__sample - type: string - - name: config__algorithm__variant_regions - type: File - - name: rgnames__pu - type: string - name: alignment_rec - type: record -outputs: -- id: align_bam - outputSource: merge_split_alignments/align_bam - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__disc - outputSource: merge_split_alignments/work_bam_plus__disc - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: work_bam_plus__sr - outputSource: merge_split_alignments/work_bam_plus__sr - secondaryFiles: - - .bai - type: - - File - - 'null' -- id: hla__fastq - outputSource: merge_split_alignments/hla__fastq - type: - - 'null' - - items: File - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: prep_align_inputs - in: - - id: alignment_rec - source: alignment_rec - out: - - id: process_alignment_rec - run: steps/prep_align_inputs.cwl -- id: process_alignment - in: - - id: alignment_rec - source: alignment_rec - - id: process_alignment_rec - source: prep_align_inputs/process_alignment_rec - out: - - id: work_bam - - id: align_bam - - id: hla__fastq - - id: work_bam_plus__disc - - id: work_bam_plus__sr - run: steps/process_alignment.cwl - scatter: - - process_alignment_rec - scatterMethod: dotproduct -- id: merge_split_alignments - in: - - id: alignment_rec - source: alignment_rec - - id: work_bam - source: process_alignment/work_bam - - id: align_bam_toolinput - source: process_alignment/align_bam - - id: work_bam_plus__disc_toolinput - source: process_alignment/work_bam_plus__disc - - id: work_bam_plus__sr_toolinput - source: process_alignment/work_bam_plus__sr - - id: hla__fastq_toolinput - source: process_alignment/hla__fastq - out: - - id: align_bam - - id: work_bam_plus__disc - - id: work_bam_plus__sr - - id: hla__fastq - run: steps/merge_split_alignments.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-svcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-svcall.cwl deleted file mode 100644 index f8724f9b0b9..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-svcall.cwl +++ /dev/null @@ -1,316 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: sv_batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: - - 'null' - - string - - name: reference__snpeff__GRCh37_75 - type: File - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: work_bam_plus__disc - type: - - File - - 'null' - - name: work_bam_plus__sr - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_batch_rec - type: record - type: array -outputs: -- id: sv_rec - outputSource: detect_sv/sv_rec - type: - items: - fields: - - name: sv__variantcaller - type: - - string - - 'null' - - name: sv__vrn_file - type: - - File - - 'null' - - name: sv__supplemental - type: - items: - - File - type: array - - name: svvalidate__summary - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: config__algorithm__svprioritize - type: - - 'null' - - string - - name: genome_resources__variation__gc_profile - type: - - 'null' - - string - - name: genome_build - type: string - - name: genome_resources__variation__germline_het_pon - type: - - 'null' - - string - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__svvalidate - type: - - 'null' - - string - - name: genome_resources__aliases__snpeff - type: string - - name: regions__sample_callable - type: - - File - - 'null' - - name: variants__samples - type: - items: - items: - - File - - 'null' - type: array - type: array - - name: depth__bins__normalized - type: - - File - - 'null' - - name: depth__bins__background - type: - - File - - 'null' - - name: depth__bins__target - type: - - File - - 'null' - - name: depth__bins__antitarget - type: - - File - - 'null' - - name: depth__bins__seq2c - type: - - File - - 'null' - - name: regions__bins__target - type: - - File - - 'null' - - name: regions__bins__antitarget - type: - - File - - 'null' - - name: regions__bins__gcannotated - type: - - File - - 'null' - - name: regions__bins__group - type: - - string - - 'null' - - name: reference__fasta__base - type: File - - name: config__algorithm__svcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__rnaseq__gene_bed - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: metadata__phenotype - type: string - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: config__algorithm__seq2c_bed_ready - type: - - File - - 'null' - - name: depth__variant_regions__regions - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: sv_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: detect_sv - in: - - id: sv_batch_rec - source: sv_batch_rec - out: - - id: sv_rec - run: steps/detect_sv.cwl diff --git a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-variantcall.cwl b/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-variantcall.cwl deleted file mode 100644 index fc5066d386e..00000000000 --- a/centaur/src/main/resources/integrationTestCases/cwl/bcbio/wes_chr21_test-workflow-gcp/wf-variantcall.cwl +++ /dev/null @@ -1,300 +0,0 @@ -class: Workflow -cwlVersion: v1.0 -hints: [] -inputs: -- id: batch_rec - type: - items: - fields: - - name: resources - type: string - - name: description - type: string - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: genome_resources__variation__clinvar - type: File - - name: genome_resources__variation__esp - type: File - - name: metadata__batch - type: string - - name: genome_resources__variation__lcr - type: File - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: vrn_file - type: - - 'null' - - string - - name: genome_resources__variation__train_hapmap - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: genome_resources__variation__1000g - type: File - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: genome_resources__variation__exac - type: File - - name: genome_resources__variation__gnomad_exome - type: - - 'null' - - string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: genome_resources__variation__dbsnp - type: File - - name: genome_resources__variation__polyx - type: File - - name: genome_resources__variation__encode_blacklist - type: File - - name: genome_resources__variation__cosmic - type: File - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__variation__train_indels - type: File - - name: genome_resources__aliases__snpeff - type: string - - name: align_bam - type: - - File - - 'null' - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: batch_rec - type: record - type: array -outputs: -- id: vc_rec - outputSource: compare_to_rm/vc_rec - type: - items: - fields: - - name: batch_samples - type: - - 'null' - - items: string - type: array - - name: validate__summary - type: - - File - - 'null' - - name: validate__tp - type: - - File - - 'null' - - name: validate__fp - type: - - File - - 'null' - - name: validate__fn - type: - - File - - 'null' - - name: resources - type: string - - name: description - type: string - - name: vrn_file - type: File - - name: reference__fasta__base - type: File - - name: config__algorithm__vcfanno - type: - - 'null' - - items: 'null' - type: array - - name: config__algorithm__variantcaller - type: string - - name: config__algorithm__coverage_interval - type: - - string - - 'null' - - name: metadata__batch - type: string - - name: config__algorithm__min_allele_fraction - type: double - - name: reference__snpeff__GRCh37_75 - type: File - - name: reference__genome_context - type: - items: File - type: array - - name: reference__rtg - type: File - - name: config__algorithm__validate - type: - - File - - 'null' - - name: config__algorithm__validate_regions - type: - - File - - 'null' - - name: genome_build - type: string - - name: metadata__phenotype - type: string - - name: genome_resources__aliases__human - type: - - string - - 'null' - - boolean - - name: config__algorithm__tools_off - type: - items: string - type: array - - name: config__algorithm__ensemble - type: string - - name: analysis - type: string - - name: config__algorithm__tools_on - type: - items: string - type: array - - name: config__algorithm__effects - type: string - - name: config__algorithm__variant_regions - type: - - File - - 'null' - - name: genome_resources__aliases__ensembl - type: string - - name: config__algorithm__exclude_regions - type: - - 'null' - - items: 'null' - type: array - - name: genome_resources__aliases__snpeff - type: string - - name: config__algorithm__variant_regions_merged - type: - - File - - 'null' - - name: regions__sample_callable - type: - - File - - 'null' - - name: config__algorithm__callable_regions - type: File - name: vc_rec - type: record - type: array -requirements: -- class: EnvVarRequirement - envDef: - - envName: MPLCONFIGDIR - envValue: . -- class: ScatterFeatureRequirement -- class: SubworkflowFeatureRequirement -steps: -- id: get_parallel_regions - in: - - id: batch_rec - source: batch_rec - out: - - id: region_block - run: steps/get_parallel_regions.cwl -- id: variantcall_batch_region - in: - - id: batch_rec - source: batch_rec - - id: region_block_toolinput - source: get_parallel_regions/region_block - out: - - id: vrn_file_region - - id: region_block - run: steps/variantcall_batch_region.cwl - scatter: - - region_block_toolinput - scatterMethod: dotproduct -- id: concat_batch_variantcalls - in: - - id: batch_rec - source: batch_rec - - id: region_block - source: variantcall_batch_region/region_block - - id: vrn_file_region - source: variantcall_batch_region/vrn_file_region - out: - - id: vrn_file - run: steps/concat_batch_variantcalls.cwl -- id: postprocess_variants - in: - - id: batch_rec - source: batch_rec - - id: vrn_file_toolinput - source: concat_batch_variantcalls/vrn_file - out: - - id: vrn_file - run: steps/postprocess_variants.cwl -- id: compare_to_rm - in: - - id: batch_rec - source: batch_rec - - id: vrn_file - source: postprocess_variants/vrn_file - out: - - id: vc_rec - run: steps/compare_to_rm.cwl diff --git a/centaur/src/main/resources/reference.conf b/centaur/src/main/resources/reference.conf index a0428e01235..d570b5beef3 100644 --- a/centaur/src/main/resources/reference.conf +++ b/centaur/src/main/resources/reference.conf @@ -86,5 +86,11 @@ centaur { include "centaur_aws_credentials.conf" } + azure { + container: "test-blob" + endpoint: "https://centaurtesting.blob.core.windows.net" + subscription: "62b22893-6bc1-46d9-8a90-806bb3cce3c9" + } + log-request-failures = false } diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.inline_file.test b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.inline_file.test deleted file mode 100644 index 98ddd1a832e..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.inline_file.test +++ /dev/null @@ -1,16 +0,0 @@ -name: inline_file -testFormat: workflowsuccess -workflowType: CWL -workflowRoot: iwdr_inline_file -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: InitialWorkDirRequirement/inline_file.cwl -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.iwdr_inline_file.prime_list": "[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97]" -} diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.inline_file_custom_entryname.test b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.inline_file_custom_entryname.test deleted file mode 100644 index 1164b6fdf2f..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.inline_file_custom_entryname.test +++ /dev/null @@ -1,16 +0,0 @@ -name: inline_file_custom_entryname -testFormat: workflowsuccess -workflowType: CWL -workflowRoot: iwdr_inline_file -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: InitialWorkDirRequirement/inline_file_custom_entryname.cwl -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.iwdr_inline_file.prime_list": "[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97]" -} diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.input_string.test b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.input_string.test deleted file mode 100644 index 4ec70064ffc..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.input_string.test +++ /dev/null @@ -1,16 +0,0 @@ -name: iwdr_input_string -testFormat: workflowsuccess -workflowType: CWL -workflowRoot: iwdr_input_string -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: InitialWorkDirRequirement/input_string.cwl -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.iwdr_input_string.prime_list": "[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97]" -} diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.input_string_function.test b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.input_string_function.test deleted file mode 100644 index 1c7973b96d6..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement.input_string_function.test +++ /dev/null @@ -1,16 +0,0 @@ -name: iwdr_input_string_function -testFormat: workflowsuccess -workflowType: CWL -workflowRoot: iwdr_input_string_function -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: InitialWorkDirRequirement/input_string_function.cwl -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.iwdr_input_string_function.prime_list": "[2, 3, 5, 7, 11, 13, 17, 19, 23, 29, 31, 37, 41, 43, 47, 53, 59, 61, 67, 71, 73, 79, 83, 89, 97]" -} diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/file_array_listing.cwl b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/file_array_listing.cwl deleted file mode 100644 index 86fb7c060a3..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/file_array_listing.cwl +++ /dev/null @@ -1,26 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: iwdr_array_listing - class: CommandLineTool - baseCommand: ['ls'] - requirements: - - class: DockerRequirement - dockerPull: "python:3.5.0" - - class: InitialWorkDirRequirement - listing: $(inputs.files) - - stdout: file_list - inputs: - - id: files - type: - type: array - items: File - default: [ "/Users/chrisl/Downloads/cwl/allRequirements.txt" ] - arguments: [ "*.txt" ] - outputs: - - id: file_list - type: string - outputBinding: - glob: file_list - loadContents: true - outputEval: $(self[0].contents.trim()) diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/inline_file.cwl b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/inline_file.cwl deleted file mode 100644 index 1980beb52d8..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/inline_file.cwl +++ /dev/null @@ -1,43 +0,0 @@ -# Little tool that uses a prime sieve to generate primes up to 100 - -cwlVersion: v1.0 -$graph: -- id: iwdr_inline_file - class: CommandLineTool - baseCommand: ['python3', 'prime_sieve.py', '100'] - requirements: - - class: DockerRequirement - dockerPull: "python:3.5.0" - - class: InitialWorkDirRequirement - listing: - - entryname: 'prime_sieve.py' - entry: | - import sys - import math - - limit = int(sys.argv[1]) - sieve = [True for i in range(limit)] - for i in range(2, math.floor(limit / 2)): - if sieve[i]: - for j in range(i * 2, limit, i): - sieve[j] = False - - result = "[" - for i in range(2, limit): - if sieve[i]: - if result != "[": - result += ", " - result += str(i) - result += "]" - - print(result) - - stdout: "primes" - inputs: [] - outputs: - - id: prime_list - type: string - outputBinding: - glob: primes - loadContents: true - outputEval: $(self[0].contents.trim()) diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/inline_file_custom_entryname.cwl b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/inline_file_custom_entryname.cwl deleted file mode 100644 index bff672a7e8a..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/inline_file_custom_entryname.cwl +++ /dev/null @@ -1,47 +0,0 @@ -# Little tool that uses a prime sieve to generate primes up to 100 - -cwlVersion: v1.0 -$graph: -- id: iwdr_inline_file - class: CommandLineTool - baseCommand: ['python3'] - requirements: - - class: DockerRequirement - dockerPull: "python:3.5.0" - - class: InitialWorkDirRequirement - listing: - - entryname: $(inputs.script_name) - entry: | - import sys - import math - - limit = int(sys.argv[1]) - sieve = [True for i in range(limit)] - for i in range(2, math.floor(limit / 2)): - if sieve[i]: - for j in range(i * 2, limit, i): - sieve[j] = False - - result = "[" - for i in range(2, limit): - if sieve[i]: - if result != "[": - result += ", " - result += str(i) - result += "]" - - print(result) - - stdout: "primes" - inputs: - - id: script_name - type: string - default: "astounding.py" - arguments: [ $(inputs.script_name), '100' ] - outputs: - - id: prime_list - type: string - outputBinding: - glob: primes - loadContents: true - outputEval: $(self[0].contents.trim()) diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/input_string.cwl b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/input_string.cwl deleted file mode 100644 index 286f6b7aedc..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/input_string.cwl +++ /dev/null @@ -1,50 +0,0 @@ -# Little tool that uses a prime sieve to generate primes up to 100 - -cwlVersion: v1.0 -$graph: -- id: iwdr_input_string - class: CommandLineTool - baseCommand: ['python3', 'prime_sieve.py', '100'] - requirements: - - class: DockerRequirement - dockerPull: "python:3.5.0" - - class: InitialWorkDirRequirement - listing: - - entryname: $(inputs.script_name) - entry: $(inputs.script) - - stdout: "primes" - inputs: - - id: script - type: string - default: | - import sys - import math - - limit = int(sys.argv[1]) - sieve = [True for i in range(limit)] - for i in range(2, math.floor(limit / 2)): - if sieve[i]: - for j in range(i * 2, limit, i): - sieve[j] = False - - result = "[" - for i in range(2, limit): - if sieve[i]: - if result != "[": - result += ", " - result += str(i) - result += "]" - - print(result) - - id: script_name - type: string - default: "prime_sieve.py" - arguments: [ $(inputs.script_name), '100' ] - outputs: - - id: prime_list - type: string - outputBinding: - glob: primes - loadContents: true - outputEval: $(self[0].contents.trim()) diff --git a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/input_string_function.cwl b/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/input_string_function.cwl deleted file mode 100644 index f70cd7bf8b9..00000000000 --- a/centaur/src/main/resources/standardTestCases/InitialWorkDirRequirement/input_string_function.cwl +++ /dev/null @@ -1,49 +0,0 @@ -# Little tool that uses a prime sieve to generate primes up to 100 - -cwlVersion: v1.0 -$graph: -- id: iwdr_input_string_function - class: CommandLineTool - baseCommand: ['python3'] - requirements: - - class: DockerRequirement - dockerPull: "python:3.5.0" - - class: InitialWorkDirRequirement - listing: - - entryname: ${var result = inputs.script_name + ".py"; return result;} - entry: ${var scr = inputs.script; return scr + "print(result);"} - - stdout: "primes" - inputs: - - id: script - type: string - default: | - import sys - import math - - limit = int(sys.argv[1]) - sieve = [True for i in range(limit)] - for i in range(2, math.floor(limit / 2)): - if sieve[i]: - for j in range(i * 2, limit, i): - sieve[j] = False - - result = "[" - for i in range(2, limit): - if sieve[i]: - if result != "[": - result += ", " - result += str(i) - result += "]" - - - id: script_name - type: string - default: "prime_sieve.py" - arguments: [ '${var result = inputs.script_name + ".py"; return result;}', '100' ] - outputs: - - id: prime_list - type: string - outputBinding: - glob: primes - loadContents: true - outputEval: $(self[0].contents.trim()) diff --git a/centaur/src/main/resources/standardTestCases/ad_hoc_file_test.aws.test b/centaur/src/main/resources/standardTestCases/ad_hoc_file_test.aws.test deleted file mode 100644 index 8016f90f665..00000000000 --- a/centaur/src/main/resources/standardTestCases/ad_hoc_file_test.aws.test +++ /dev/null @@ -1,14 +0,0 @@ -name: ad_hoc_file_test.aws -testFormat: workflowsuccess -backends: [ AWSBATCH ] - -files { - workflow: ad_hoc_file_test/workflow.cwl - imports: [ - ad_hoc_file_test/cwl-test.cwl - ] -} - -metadata { - status: Succeeded -} diff --git a/centaur/src/main/resources/standardTestCases/ad_hoc_file_test/cwl-test.cwl b/centaur/src/main/resources/standardTestCases/ad_hoc_file_test/cwl-test.cwl deleted file mode 100644 index 7ac312f6f3a..00000000000 --- a/centaur/src/main/resources/standardTestCases/ad_hoc_file_test/cwl-test.cwl +++ /dev/null @@ -1,21 +0,0 @@ -class: CommandLineTool -cwlVersion: v1.0 -baseCommand: ["sh", "example.sh"] -hints: - DockerRequirement: - dockerPull: ubuntu:latest -inputs: [] - -requirements: - InitialWorkDirRequirement: - listing: - - entryname: example.sh - entry: |- - PREFIX='Message is:' - MSG="\${PREFIX} Hello world!" - echo \${MSG} - -outputs: - example_out: - type: stdout -stdout: output.txt diff --git a/centaur/src/main/resources/standardTestCases/ad_hoc_file_test/workflow.cwl b/centaur/src/main/resources/standardTestCases/ad_hoc_file_test/workflow.cwl deleted file mode 100644 index 678132165b2..00000000000 --- a/centaur/src/main/resources/standardTestCases/ad_hoc_file_test/workflow.cwl +++ /dev/null @@ -1,10 +0,0 @@ -cwlVersion: v1.0 -class: Workflow -inputs: [] -outputs: [] - -steps: - test: - run: cwl-test.cwl - in: [] - out: [] diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test b/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test index 184d41b5aeb..b9f5430e57b 100644 --- a/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_engine_functions.test @@ -54,4 +54,40 @@ metadata { "outputs.biscayne_new_engine_functions.bigIntFloatComparison": 10.0 "outputs.biscayne_new_engine_functions.minMaxIntFloatComposition": 1.0 "outputs.biscayne_new_engine_functions.maxIntVsMaxFloat": 1.79769313E+308 + + "outputs.biscayne_new_engine_functions.substituted": "WATtheWAT" + + "outputs.biscayne_new_engine_functions.with_suffixes.0": "aaaS" + "outputs.biscayne_new_engine_functions.with_suffixes.1": "bbbS" + "outputs.biscayne_new_engine_functions.with_suffixes.2": "cccS" + + "outputs.biscayne_new_engine_functions.with_quotes.0": "\"1\"" + "outputs.biscayne_new_engine_functions.with_quotes.1": "\"2\"" + "outputs.biscayne_new_engine_functions.with_quotes.2": "\"3\"" + + "outputs.biscayne_new_engine_functions.string_with_quotes.0": "\"aaa\"" + "outputs.biscayne_new_engine_functions.string_with_quotes.1": "\"bbb\"" + "outputs.biscayne_new_engine_functions.string_with_quotes.2": "\"ccc\"" + + "outputs.biscayne_new_engine_functions.with_squotes.0": "'1'" + "outputs.biscayne_new_engine_functions.with_squotes.1": "'2'" + "outputs.biscayne_new_engine_functions.with_squotes.2": "'3'" + + "outputs.biscayne_new_engine_functions.string_with_squotes.0": "'aaa'" + "outputs.biscayne_new_engine_functions.string_with_squotes.1": "'bbb'" + "outputs.biscayne_new_engine_functions.string_with_squotes.2": "'ccc'" + "outputs.biscayne_new_engine_functions.unzipped_a.left.0": "A" + "outputs.biscayne_new_engine_functions.unzipped_a.right.0": "a" + + "outputs.biscayne_new_engine_functions.unzipped_b.left.0": "A" + "outputs.biscayne_new_engine_functions.unzipped_b.left.1": "B" + "outputs.biscayne_new_engine_functions.unzipped_b.right.0": "a" + "outputs.biscayne_new_engine_functions.unzipped_b.right.1": "b" + + "outputs.biscayne_new_engine_functions.unzipped_c.left.0": "one" + "outputs.biscayne_new_engine_functions.unzipped_c.left.1": "two" + "outputs.biscayne_new_engine_functions.unzipped_c.left.2": "three" + "outputs.biscayne_new_engine_functions.unzipped_c.right.0": 1.0 + "outputs.biscayne_new_engine_functions.unzipped_c.right.1": 2.0 + "outputs.biscayne_new_engine_functions.unzipped_c.right.2": 3.0 } diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_lifesciences.test b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_lifesciences.test new file mode 100644 index 00000000000..7fcb3fb4c28 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_lifesciences.test @@ -0,0 +1,17 @@ +name: biscayne_new_runtime_attributes_lifesciences +testFormat: workflowsuccess +tags: ["wdl_biscayne"] + +# Will run on a Cromwell that supports any one of these backends +backendsMode: any +backends: [Papi, Papiv2, GCPBatch] + +files { + workflow: wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl +} + +metadata { + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.docker": "rockylinux:9", + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.cpu": 4 + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.memory": "6 GB" +} diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_local.test b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_local.test new file mode 100644 index 00000000000..d25d0f7e59b --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_local.test @@ -0,0 +1,16 @@ +name: biscayne_new_runtime_attributes_local +testFormat: workflowsuccess +tags: ["wdl_biscayne"] + +# This test should only run in the Local suite, on its default `Local` backend. Unfortunately the `Local` backend +# leaks into other suites, so require an irrelevant `LocalNoDocker` backend that is only found in Local suite. +backends: [Local, LocalNoDocker] + +files { + workflow: wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl +} + +# CPU, memory attributes not applicable for Local backend +metadata { + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.docker": "ubuntu:latest", +} diff --git a/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_tes.test b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_tes.test new file mode 100644 index 00000000000..8ee9cf96050 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/biscayne_new_runtime_attributes_tes.test @@ -0,0 +1,14 @@ +name: biscayne_new_runtime_attributes_tes +testFormat: workflowsuccess +tags: ["wdl_biscayne"] +backends: [TES] + +files { + workflow: wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl +} + +metadata { + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.docker": "debian:latest", + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.cpu": 4 + "calls.runtime_attributes_wf.runtime_attributes_task.runtimeAttributes.memory": "4 GB" +} diff --git a/centaur/src/main/resources/standardTestCases/biscayne_prohibits_directory.test b/centaur/src/main/resources/standardTestCases/biscayne_prohibits_directory.test new file mode 100644 index 00000000000..d2d9bfc9ddf --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/biscayne_prohibits_directory.test @@ -0,0 +1,13 @@ +name: directory_type_local_denied +testFormat: workflowfailure +tags: [localdockertest, "wdl_biscayne"] +backends: [Local, LocalNoDocker] + +files { + workflow: wdl_biscayne/biscayne_prohibits_directory/directory_type.wdl + inputs: wdl_biscayne/biscayne_prohibits_directory/directory_type_local_inputs.json +} + +metadata { + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json new file mode 100644 index 00000000000..f004c8be647 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.inputs.json @@ -0,0 +1,3 @@ +{ + "fileChecksum.inputFile": "https://.blob.core.windows.net/cromwell/user-inputs/inputFile.txt" +} diff --git a/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.wdl b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.wdl new file mode 100644 index 00000000000..b482f80d6e4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/blob_md5/blob_md5.wdl @@ -0,0 +1,20 @@ +task md5 { + File inputFile + command { + echo "`date`: Running checksum on ${inputFile}..." + md5sum ${inputFile} > md5sum.txt + echo "`date`: Checksum is complete." + } + output { + File result = "md5sum.txt" + } + runtime { + docker: 'ubuntu:18.04' + preemptible: true + } +} + +workflow fileChecksum { + File inputFile + call md5 { input: inputFile=inputFile} +} diff --git a/centaur/src/main/resources/standardTestCases/composedenginefunctions.test b/centaur/src/main/resources/standardTestCases/composedenginefunctions.test index 68628483740..d51f31c9929 100644 --- a/centaur/src/main/resources/standardTestCases/composedenginefunctions.test +++ b/centaur/src/main/resources/standardTestCases/composedenginefunctions.test @@ -1,6 +1,5 @@ name: composedenginefunctions testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: composedenginefunctions/composedenginefunctions.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test b/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test index 2bd93388199..9d20530291a 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.ifs_in_scatters.test @@ -1,6 +1,6 @@ name: ifs_in_scatters testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/ifs_in_scatters/ifs_in_scatters.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test b/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test index 6f5c0fa2c93..cf4ca22c876 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.nested_lookups.test @@ -1,6 +1,6 @@ name: nested_lookups testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/nested_lookups/nested_lookups.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test b/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test index 8d9cc2193b7..f23865eae67 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.scatters_in_ifs.test @@ -1,6 +1,6 @@ name: scatters_in_ifs testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/scatters_in_ifs/scatters_in_ifs.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test b/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test index 82794b863d5..50a1c811892 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.simple_if.test @@ -1,6 +1,6 @@ name: simple_if testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/simple_if/simple_if.wdl diff --git a/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test b/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test index 3aed74d756b..ede32957e47 100644 --- a/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test +++ b/centaur/src/main/resources/standardTestCases/conditionals.simple_if_workflow_outputs.test @@ -1,6 +1,6 @@ name: simple_if_workflow_outputs testFormat: workflowsuccess -tags: [ conditionals, "wdl_upgrade" ] +tags: [ conditionals ] files { workflow: conditionals_tests/simple_if_workflow_outputs/simple_if_workflow_outputs.wdl diff --git a/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows.test b/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows.test deleted file mode 100644 index e5381017829..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows.test +++ /dev/null @@ -1,19 +0,0 @@ -name: cwl_cache_between_workflows -testFormat: runtwiceexpectingcallcaching -workflowRoot: cwl-cache-between-workflows -workflowType: CWL -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: cwl_cache_between_workflows/cwl_cache_between_workflows.cwl - inputs: cwl_cache_between_workflows/cwl_cache_between_workflows.json -} - -metadata { - status: Succeeded - "calls.cwl-cache-between-workflows.step-average.callCaching.result": "Cache Hit: <>:cwl-cache-between-workflows.step-average:-1" - "calls.cwl-cache-between-workflows.step-product.callCaching.result": "Cache Hit: <>:cwl-cache-between-workflows.step-product:-1" - "outputs.cwl-cache-between-workflows.trapezoidalArea": 77.0, - "outputs.cwl-cache-between-workflows.baseAverage": 11.0 -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows/cwl_cache_between_workflows.cwl b/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows/cwl_cache_between_workflows.cwl deleted file mode 100644 index d48e74cd935..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows/cwl_cache_between_workflows.cwl +++ /dev/null @@ -1,79 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: average - class: CommandLineTool - - inputs: - base1: - type: int - default: 9 - base2: - type: int - default: 13 - - outputs: - baseAverage: - type: float - outputBinding: - glob: stdout.txt - loadContents: true - outputEval: $(parseFloat(self[0].contents)) - - arguments: - - valueFrom: "echo $((inputs.base1 + inputs.base2) / 2)" - shellQuote: false - - stdout: stdout.txt - -- id: height-product - class: CommandLineTool - - inputs: - baseAverage: float - height: - type: float - default: 7.0 - - outputs: - trapezoidalArea: - type: long - outputBinding: - glob: stdout.txt - loadContents: true - outputEval: $(parseFloat(self[0].contents)) - - arguments: - - valueFrom: "echo $(inputs.baseAverage * inputs.height)" - shellQuote: false - - stdout: stdout.txt - -- id: cwl-cache-between-workflows - class: Workflow - - requirements: - - class: DockerRequirement - dockerPull: "ubuntu:latest" - - class: ShellCommandRequirement - - inputs: [] - - outputs: - baseAverage: - type: float - outputSource: step-average/baseAverage - trapezoidalArea: - type: long - outputSource: step-product/trapezoidalArea - - steps: - step-average: - run: "#average" - in: [] - out: [baseAverage] - - step-product: - run: "#height-product" - in: - baseAverage: step-average/baseAverage - out: [trapezoidalArea] diff --git a/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows/cwl_cache_between_workflows.json b/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows/cwl_cache_between_workflows.json deleted file mode 100644 index 0967ef424bc..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_cache_between_workflows/cwl_cache_between_workflows.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow.test b/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow.test deleted file mode 100644 index 8ef753b494d..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow.test +++ /dev/null @@ -1,26 +0,0 @@ -name: cwl_cache_within_workflow -testFormat: workflowsuccess -workflowRoot: cwl-cache-within-workflow -workflowType: CWL -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: cwl_cache_within_workflow/cwl_cache_within_workflow.cwl - inputs: cwl_cache_within_workflow/cwl_cache_within_workflow.json -} - -metadata { - status: Succeeded - "calls.cwl-cache-within-workflow.foo.callCaching.result": "Cache Miss" - "calls.cwl-cache-within-workflow.bar.callCaching.result": "Cache Miss" - "calls.cwl-cache-within-workflow.re-bar.callCaching.result": "Cache Hit: <>:cwl-cache-within-workflow.bar:-1" - "calls.cwl-cache-within-workflow.bar.inputs.pi": 3.14159 - "calls.cwl-cache-within-workflow.bar.inputs.rSquared": 16.23381791021144 - "calls.cwl-cache-within-workflow.bar.outputs.area": 51 - "calls.cwl-cache-within-workflow.bar.outputs.rSquaredCopy": 16.23381791021144 - "calls.cwl-cache-within-workflow.re-bar.inputs.pi": 3.14159 - "calls.cwl-cache-within-workflow.re-bar.inputs.rSquared": 16.23381791021144 - "calls.cwl-cache-within-workflow.re-bar.outputs.area": 51 - "calls.cwl-cache-within-workflow.re-bar.outputs.rSquaredCopy": 16.23381791021144 -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow/cwl_cache_within_workflow.cwl b/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow/cwl_cache_within_workflow.cwl deleted file mode 100644 index 15b1fd79a01..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow/cwl_cache_within_workflow.cwl +++ /dev/null @@ -1,129 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: one - class: CommandLineTool - - inputs: - r: - type: float - - outputs: - rSquared: - type: float - outputBinding: - glob: stdout.txt - loadContents: true - outputEval: $(parseFloat(self[0].contents)) - rCopy: - type: float - outputBinding: - outputEval: $(inputs.r) - - arguments: - - valueFrom: "echo $(inputs.r * inputs.r)" - shellQuote: false - - stdout: stdout.txt - -- id: two - class: CommandLineTool - - inputs: - rSquared: float - pi: float - - outputs: - area: - outputBinding: - glob: stdout.txt - loadContents: true - outputEval: $(parseInt(self[0].contents)) - type: int - rSquaredCopy: - outputBinding: - outputEval: $(inputs.rSquared) - type: float - - arguments: - - valueFrom: "echo $(inputs.rSquared * inputs.pi)" - shellQuote: false - - stdout: stdout.txt - -# Throwing in some ExpressionTools for fun. Currently these not only don't cache but aren't even listed in the metadata -# except in the original workflow blob (see #3499). The current thinking is that these should appear in metadata with a -# call-like presentation but that they don't need to be eligible for caching since ExpressionTools are supposed to be -# very lightweight and not worth the expense of hashing and potential cache hit copying. -- id: three - class: ExpressionTool - - inputs: - rSquared: float - pi: float - - outputs: - area: int - rSquaredCopy: float - - expression: | - ${ - return {"area": parseInt(inputs.pi * inputs.rSquared), - "rSquaredCopy": inputs.rSquared }; - } - - -- id: cwl-cache-within-workflow - class: Workflow - - requirements: - - class: DockerRequirement - dockerPull: "ubuntu:latest" - - class: ShellCommandRequirement - - inputs: - radius: float - pi: float - - outputs: - area: - type: int - outputSource: re-bar/area - area-expression: - type: int - outputSource: re-baz/area - - steps: - foo: - run: "#one" - in: - r: radius - pi: pi - out: [rSquared, rCopy] - - bar: - run: "#two" - in: - rSquared: "foo/rSquared" - pi: pi - out: [area, rSquaredCopy] - - re-bar: - run: "#two" - in: - rSquared: "bar/rSquaredCopy" - pi: pi - out: [area, rSquaredCopy] - - baz: - run: "#three" - in: - rSquared: "foo/rSquared" - pi: pi - out: [area, rSquaredCopy] - - re-baz: - run: "#three" - in: - rSquared: "baz/rSquaredCopy" - pi: pi - out: [area, rSquaredCopy] diff --git a/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow/cwl_cache_within_workflow.json b/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow/cwl_cache_within_workflow.json deleted file mode 100644 index 563f2413547..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_cache_within_workflow/cwl_cache_within_workflow.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "radius": 4.029121233, - "pi": 3.14159 -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_disk_resources_papiv2.test b/centaur/src/main/resources/standardTestCases/cwl_disk_resources_papiv2.test deleted file mode 100644 index 8677bcf711e..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_disk_resources_papiv2.test +++ /dev/null @@ -1,17 +0,0 @@ -name: cwl_disk_resources_papiv2 -testFormat: workflowsuccess -backends: [Papiv2] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_resources/cwl_disk_resources.cwl -} - -metadata { - status: Succeeded - "outputs.diskSizeTool.disk_size": 30 -} - -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: diskSizeTool diff --git a/centaur/src/main/resources/standardTestCases/cwl_docker_size.test b/centaur/src/main/resources/standardTestCases/cwl_docker_size.test deleted file mode 100644 index ca700d5cd9d..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_docker_size.test +++ /dev/null @@ -1,21 +0,0 @@ -name: cwl_docker_files -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: main -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_docker_size/cwl_docker_size.cwl - inputs: cwl_docker_size/cwl_docker_size.yaml - options: cwl_docker_size/cwl_docker_size.options -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.main.stdout_output": "execute order=66" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.cwl b/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.cwl deleted file mode 100644 index 5383bccb484..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.cwl +++ /dev/null @@ -1,40 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: main - class: CommandLineTool - hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" - inputs: - - id: INPUT - type: File[] - stdout: stdout - outputs: - stdout_output: - type: string - outputBinding: - glob: stdout - loadContents: true - outputEval: $(self[0].contents.trim()) - arguments: - - valueFrom: | - ${ - if (inputs.INPUT.length == 0) { - var cmd = ['echo', "no inputs"]; - return cmd - } - else { - var cmd = ["echo", "execute"]; - var use_input = []; - for (var i = 0; i < inputs.INPUT.length; i++) { - var filesize = inputs.INPUT[i].size; - use_input.push("order=".concat(filesize)); - } - - var run_cmd = cmd.concat(use_input); - return run_cmd - } - - } - out: stdout - baseCommand: [] diff --git a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.options b/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.options deleted file mode 100644 index 71d245d2bf6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.options +++ /dev/null @@ -1,3 +0,0 @@ -{ - "backend": "Local" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.yaml b/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.yaml deleted file mode 100644 index 097e69e8590..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size.yaml +++ /dev/null @@ -1,3 +0,0 @@ -INPUT: - - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size_input.txt" diff --git a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size_input.txt b/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size_input.txt deleted file mode 100644 index 1a6374205aa..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_docker_size/cwl_docker_size_input.txt +++ /dev/null @@ -1 +0,0 @@ -And you people, you're all astronauts, on some kind of star trek. diff --git a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir.test b/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir.test deleted file mode 100644 index 57a69084c62..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir.test +++ /dev/null @@ -1,20 +0,0 @@ -name: cwl_dynamic_initial_workdir -testFormat: workflowsuccess -ignore: false -workflowRoot: main -workflowType: CWL -workflowTypeVersion: v1.0 -backendsMode: "only" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.cwl - inputs: cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.inputs -} - -metadata { - status: Succeeded - "outputs.main.sha": "13cda8661796ae241da3a18668fb552161a72592\n" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.cwl b/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.cwl deleted file mode 100644 index 6012854d0b7..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.cwl +++ /dev/null @@ -1,29 +0,0 @@ -cwlVersion: v1.0 -$graph: - id: main - class: CommandLineTool - cwlVersion: v1.0 - requirements: - - class: ShellCommandRequirement - - class: InitialWorkDirRequirement - listing: $(inputs.indir.listing) - - class: DockerRequirement - dockerPull: "ubuntu:latest" - inputs: - indir: Directory - outputs: - sha: - type: string - outputBinding: - glob: output.txt - loadContents: true - outputEval: $(self[0].contents) - arguments: ["find", "-L", ".", "-name", "?", - {shellQuote: false, valueFrom: "|"}, - "sort", - {shellQuote: false, valueFrom: "|"}, - "sha1sum", - {shellQuote: false, valueFrom: "|"}, - "cut", "-c", "1-40" - ] - stdout: output.txt diff --git a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.inputs b/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.inputs deleted file mode 100644 index f37fed14367..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/cwl_dynamic_initial_workdir.inputs +++ /dev/null @@ -1,3 +0,0 @@ -indir: - class: Directory - location: centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/testdir diff --git a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/testdir/a b/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/testdir/a deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/testdir/b b/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/testdir/b deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/testdir/c/d b/centaur/src/main/resources/standardTestCases/cwl_dynamic_initial_workdir/testdir/c/d deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format.cwl b/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format.cwl deleted file mode 100644 index c2079a82721..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format.cwl +++ /dev/null @@ -1,25 +0,0 @@ -cwlVersion: v1.0 -$namespaces: - edam: http://edamontology.org/ - gx: http://galaxyproject.org/formats/ -$schemas: - - file:./cwl/src/test/resources/cwl/ontology/EDAM.owl - - file:./cwl/src/test/resources/cwl/ontology/gx_edam.ttl -$graph: -- id: main - class: CommandLineTool - cwlVersion: v1.0 - inputs: - - id: reference - type: File - format: gx:fasta - inputBinding: { position: 2 } - outputs: - - id: lineCount - type: int - outputBinding: - glob: stdout - loadContents: true - outputEval: "$(parseInt(self[0].contents))" - baseCommand: ["wc", "-l"] - stdout: stdout diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format.options b/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format.options deleted file mode 100644 index 71d245d2bf6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format.options +++ /dev/null @@ -1,3 +0,0 @@ -{ - "backend": "Local" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_invalid.yaml b/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_invalid.yaml deleted file mode 100644 index 6e75ae34f28..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_invalid.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Try to pass a fastq in as a fasta -reference: - class: File - format: edam:format_1930 - path: "centaur/src/main/resources/standardTestCases/cwl_format/foo.fastq" diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_missing.yaml b/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_missing.yaml deleted file mode 100644 index 4e8a35cb2e9..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_missing.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Pass a fastq in as a fasta, but without passing the format -reference: - class: File - #format: edam:format_1930 - path: "centaur/src/main/resources/standardTestCases/cwl_format/foo.fastq" diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_url.cwl b/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_url.cwl deleted file mode 100644 index b85bf8d2434..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_url.cwl +++ /dev/null @@ -1,27 +0,0 @@ -cwlVersion: v1.0 -$namespaces: - edam: http://edamontology.org/ - gx: http://galaxyproject.org/formats/ -$schemas: - # Relative path: - - ../../../../../../cwl/src/test/resources/cwl/ontology/EDAM.owl - # Absolute (remote) path: - - https://raw.githubusercontent.com/broadinstitute/cromwell/develop/cwl/src/test/resources/cwl/ontology/gx_edam.ttl -$graph: -- id: main - class: CommandLineTool - cwlVersion: v1.0 - inputs: - - id: reference - type: File - format: gx:fasta - inputBinding: { position: 2 } - outputs: - - id: lineCount - type: int - outputBinding: - glob: stdout - loadContents: true - outputEval: "$(parseInt(self[0].contents))" - baseCommand: ["wc", "-l"] - stdout: stdout diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_valid.yaml b/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_valid.yaml deleted file mode 100644 index b598f92b7e1..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_valid.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# Pass in a valid input file format -reference: - class: File - format: edam:format_1929 - path: "centaur/src/main/resources/standardTestCases/cwl_format/foo.fasta" diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/foo.fasta b/centaur/src/main/resources/standardTestCases/cwl_format/foo.fasta deleted file mode 100644 index e91d1521810..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/foo.fasta +++ /dev/null @@ -1,2 +0,0 @@ ->sequence #1 -GATTTGGGGTTCAAAGCAGTATCGATCAAATAGTAAATCCATTTGTTCAACTCACAGTTT diff --git a/centaur/src/main/resources/standardTestCases/cwl_format/foo.fastq b/centaur/src/main/resources/standardTestCases/cwl_format/foo.fastq deleted file mode 100644 index 5ecec0ebec8..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format/foo.fastq +++ /dev/null @@ -1,4 +0,0 @@ -@SEQ_ID -GATTTGGGGTTCAAAGCAGTATCGATCAAATAGTAAATCCATTTGTTCAACTCACAGTTT -+ -!''*((((***+))%%%++)(%%%%).1***-+*''))**55CCF>>>>>>CCCCCCC65 diff --git a/centaur/src/main/resources/standardTestCases/cwl_format_invalid.test b/centaur/src/main/resources/standardTestCases/cwl_format_invalid.test deleted file mode 100644 index 43962bb5bfc..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format_invalid.test +++ /dev/null @@ -1,22 +0,0 @@ -name: cwl_format_invalid -testFormat: workflowfailure -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: main -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_format/cwl_format.cwl - inputs: cwl_format/cwl_format_invalid.yaml - options: cwl_format/cwl_format.options -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "failures.0.message": "Workflow input processing failed" - "failures.0.causedBy.0.message": "Failed to evaluate input 'reference' (reason 1 of 1): edam:format_1930 is not compatible with http://galaxyproject.org/formats/fasta" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_format_missing.test b/centaur/src/main/resources/standardTestCases/cwl_format_missing.test deleted file mode 100644 index a7b626f69e4..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format_missing.test +++ /dev/null @@ -1,22 +0,0 @@ -name: cwl_format_missing -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: main -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_format/cwl_format.cwl - inputs: cwl_format/cwl_format_missing.yaml - options: cwl_format/cwl_format.options -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "workflowName": "main" - "outputs.main.lineCount": "4" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_format_valid.test b/centaur/src/main/resources/standardTestCases/cwl_format_valid.test deleted file mode 100644 index 6a6452deff5..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format_valid.test +++ /dev/null @@ -1,22 +0,0 @@ -name: cwl_format_valid -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: main -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_format/cwl_format.cwl - inputs: cwl_format/cwl_format_valid.yaml - options: cwl_format/cwl_format.options -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "workflowName": "main" - "outputs.main.lineCount": "2" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_format_valid_with_workflow_url.test b/centaur/src/main/resources/standardTestCases/cwl_format_valid_with_workflow_url.test deleted file mode 100644 index e587dbfe6d6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_format_valid_with_workflow_url.test +++ /dev/null @@ -1,22 +0,0 @@ -name: cwl_format_valid_with_workflow_url -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: main -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflowUrl: "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/cwl_format/cwl_format_url.cwl" - inputs: cwl_format/cwl_format_valid.yaml - options: cwl_format/cwl_format.options -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "workflowName": "main" - "outputs.main.lineCount": "2" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_glob_sort.test b/centaur/src/main/resources/standardTestCases/cwl_glob_sort.test deleted file mode 100644 index 04856ab66b9..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_glob_sort.test +++ /dev/null @@ -1,16 +0,0 @@ -name: cwl_glob_sort -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: globSort -skipDescribeEndpointValidation: true - -files { - workflow: cwl_glob_sort/cwl_glob_sort.cwl -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.globSort.letters": "a b c w x y z" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_glob_sort/cwl_glob_sort.cwl b/centaur/src/main/resources/standardTestCases/cwl_glob_sort/cwl_glob_sort.cwl deleted file mode 100644 index 8d54af1c040..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_glob_sort/cwl_glob_sort.cwl +++ /dev/null @@ -1,19 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: globSort - cwlVersion: v1.0 - class: CommandLineTool - requirements: - - class: InlineJavascriptRequirement - hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" - inputs: [] - baseCommand: [touch, z, y, x, w, c, b, a] - outputs: - letters: - type: string - outputBinding: - glob: '?' - outputEval: | - ${ return self.sort(function(a,b) { return a.location > b.location ? 1 : (a.location < b.location ? -1 : 0) }).map(f => f.basename).join(" ") } diff --git a/centaur/src/main/resources/standardTestCases/cwl_glob_sort_with_workflow_url.test b/centaur/src/main/resources/standardTestCases/cwl_glob_sort_with_workflow_url.test deleted file mode 100644 index 6e7ed2b1dd2..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_glob_sort_with_workflow_url.test +++ /dev/null @@ -1,16 +0,0 @@ -name: cwl_glob_sort_with_workflow_url -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: globSort -skipDescribeEndpointValidation: true - -files { - workflowUrl: "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/cwl_glob_sort/cwl_glob_sort.cwl" -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.globSort.letters": "a b c w x y z" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type.test b/centaur/src/main/resources/standardTestCases/cwl_import_type.test deleted file mode 100644 index 9e530a95469..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type.test +++ /dev/null @@ -1,14 +0,0 @@ -name: cwl_import_type -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: cwl_import_type/test_wf.cwl - inputs: cwl_import_type/test_wf_inputs.json - imports: [ - cwl_import_type/capture_kit.yml, - cwl_import_type/touch.cwl - ] -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type/capture_kit.yml b/centaur/src/main/resources/standardTestCases/cwl_import_type/capture_kit.yml deleted file mode 100644 index 4323dd90f6d..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type/capture_kit.yml +++ /dev/null @@ -1,5 +0,0 @@ -- name: capture_kit - type: record - fields: - - name: bait - type: string diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type/test_wf.cwl b/centaur/src/main/resources/standardTestCases/cwl_import_type/test_wf.cwl deleted file mode 100644 index 3658ab127d0..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type/test_wf.cwl +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 - -class: Workflow - -requirements: - - class: SchemaDefRequirement - types: - - $import: capture_kit.yml - -inputs: - - id: bam - type: string - - id: capture_kit - type: capture_kit.yml#capture_kit - -outputs: - - id: output_bam - type: File - outputSource: touch_bam/output - -steps: - - id: touch_bam - run: touch.cwl - in: - - id: input - source: bam - out: - - id: output diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type/test_wf_inputs.json b/centaur/src/main/resources/standardTestCases/cwl_import_type/test_wf_inputs.json deleted file mode 100644 index 55141261811..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type/test_wf_inputs.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "bam": "a.bam", - "capture_kit": { - "bait": "abait" - } -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type/touch.cwl b/centaur/src/main/resources/standardTestCases/cwl_import_type/touch.cwl deleted file mode 100644 index 8a0be057c42..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type/touch.cwl +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 - -requirements: - - class: DockerRequirement - dockerPull: ubuntu:bionic-20180426 - -class: CommandLineTool - -inputs: - - id: input - type: string - inputBinding: - position: 0 - -outputs: - - id: output - type: File - outputBinding: - glob: $(inputs.input) - -baseCommand: [touch] diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type_packed.test b/centaur/src/main/resources/standardTestCases/cwl_import_type_packed.test deleted file mode 100644 index 0a3fd7c20a4..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type_packed.test +++ /dev/null @@ -1,11 +0,0 @@ -name: cwl_import_type_packed -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: main -skipDescribeEndpointValidation: true - -files { - workflow: cwl_import_type_packed/test_pack.cwl - inputs: cwl_import_type_packed/test_wf_inputs.json -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type_packed/test_pack.cwl b/centaur/src/main/resources/standardTestCases/cwl_import_type_packed/test_pack.cwl deleted file mode 100644 index 4150b5a5796..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type_packed/test_pack.cwl +++ /dev/null @@ -1,91 +0,0 @@ -{ - "cwlVersion": "v1.0", - "$graph": [ - { - "inputs": [ - { - "type": "string", - "id": "#main/bam" - }, - { - "type": "#capture_kit.yml/capture_kit", - "id": "#main/capture_kit" - } - ], - "requirements": [ - { - "class": "SchemaDefRequirement", - "types": [ - { - "fields": [ - { - "type": "string", - "name": "#capture_kit.yml/capture_kit/bait" - } - ], - "type": "record", - "name": "#capture_kit.yml/capture_kit" - } - ] - } - ], - "outputs": [ - { - "outputSource": "#main/touch_bam/output", - "type": "File", - "id": "#main/output_bam" - } - ], - "class": "Workflow", - "steps": [ - { - "out": [ - { - "id": "#main/touch_bam/output" - } - ], - "run": "#touch.cwl", - "id": "#main/touch_bam", - "in": [ - { - "source": "#main/bam", - "id": "#main/touch_bam/input" - } - ] - } - ], - "id": "#main" - }, - { - "inputs": [ - { - "inputBinding": { - "position": 0 - }, - "type": "string", - "id": "#touch.cwl/input" - } - ], - "requirements": [ - { - "dockerPull": "ubuntu:bionic-20180426", - "class": "DockerRequirement" - } - ], - "outputs": [ - { - "outputBinding": { - "glob": "$(inputs.input)" - }, - "type": "File", - "id": "#touch.cwl/output" - } - ], - "baseCommand": [ - "touch" - ], - "class": "CommandLineTool", - "id": "#touch.cwl" - } - ] -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_import_type_packed/test_wf_inputs.json b/centaur/src/main/resources/standardTestCases/cwl_import_type_packed/test_wf_inputs.json deleted file mode 100644 index 55141261811..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_import_type_packed/test_wf_inputs.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "bam": "a.bam", - "capture_kit": { - "bait": "abait" - } -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression.test b/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression.test deleted file mode 100644 index 0e39f78ce78..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression.test +++ /dev/null @@ -1,17 +0,0 @@ -name: cwl_input_binding_expression -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: cwl_input_binding_expression -skipDescribeEndpointValidation: true - -files { - workflow: cwl_input_binding_expression/cwl_input_binding_expression.cwl - inputs: cwl_input_binding_expression/cwl_input_binding_expression.json -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.cwl_input_binding_expression.b": "hello world" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression/cwl_input_binding_expression.cwl b/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression/cwl_input_binding_expression.cwl deleted file mode 100644 index f534adfaa36..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression/cwl_input_binding_expression.cwl +++ /dev/null @@ -1,25 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: cwl_input_binding_expression - class: CommandLineTool - requirements: - - class: InlineJavascriptRequirement - hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" - inputs: - - id: hello - type: string - inputBinding: - valueFrom: $(self + " world") - position: 1 - outputs: - b: - type: string - outputBinding: - loadContents: true - glob: stdout - outputEval: $(self[0].contents.trim()) - stdout: stdout - baseCommand: [] - arguments: [echo] diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression/cwl_input_binding_expression.json b/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression/cwl_input_binding_expression.json deleted file mode 100644 index 8920ee8d630..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_binding_expression/cwl_input_binding_expression.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "hello": "hello" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_json.test b/centaur/src/main/resources/standardTestCases/cwl_input_json.test deleted file mode 100644 index 1dd81ba8260..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_json.test +++ /dev/null @@ -1,18 +0,0 @@ -name: cwl_input_json -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -backendsMode: "only" -workflowRoot: cwl_input_json -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_input_json/cwl_input_json.yaml -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_json/cwl_input_json.yaml b/centaur/src/main/resources/standardTestCases/cwl_input_json/cwl_input_json.yaml deleted file mode 100644 index f6e5616c6c4..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_json/cwl_input_json.yaml +++ /dev/null @@ -1,70 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: cwl_input_json - class: Workflow - inputs: [] - outputs: - final_output: - type: File - outputSource: round/output_file - steps: - make: - run: "#makefile" - in: [] - out: [fileoutput] - round: - run: "#roundtrip" - in: - input_record: - source: "make/fileoutput" - out: [output_file] - -- id: makefile - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - - class: InlineJavascriptRequirement - - class: DockerRequirement - dockerPull: "ubuntu:latest" - inputs: [] - outputs: - fileoutput: - type: - fields: - - name: input_file - type: File - name: input_record - type: record - arguments: - - valueFrom: > - echo foo > foo && echo '{ "fileoutput": { "input_file": {"path": "$(runtime.outdir)/foo", "class": "File"} } }' > cwl.output.json - shellQuote: false - -- id: roundtrip - class: CommandLineTool - hints: - - class: DockerRequirement - dockerPull: "stedolan/jq:latest" - inputs: - - id: input_record - type: - fields: - - name: input_file - type: File - name: input_record - type: record - outputs: - - id: output_file - type: File - requirements: - - class: ShellCommandRequirement - - class: InlineJavascriptRequirement - - class: InitialWorkDirRequirement - listing: - - entry: $(JSON.stringify(inputs)) - entryname: cwl.inputs.json - arguments: - # Round-trips the file referenced in cwl.input.json to cwl.output.json. Also ls it in the command to make sure it's there. - - valueFrom: > - INPUT_FILE=\$(cat cwl.inputs.json | jq -r '.. | .path? // empty') && ls $INPUT_FILE && echo "{\"output_file\": {\"path\": \"\$INPUT_FILE\", \"class\": \"File\"} }" > cwl.output.json - shellQuote: false diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_typearray.test b/centaur/src/main/resources/standardTestCases/cwl_input_typearray.test deleted file mode 100644 index a87bcbf9f65..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_typearray.test +++ /dev/null @@ -1,18 +0,0 @@ -name: cwl_input_typearray -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: input_typearray -skipDescribeEndpointValidation: true - -files { - workflow: cwl_input_typearray/input_typearray.cwl - inputs: cwl_input_typearray/input_typearray.yml -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.input_typearray.response_f": "input.txt" - "outputs.input_typearray.response_s": "nonexistent_path.txt" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input.txt b/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input_typearray.cwl b/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input_typearray.cwl deleted file mode 100644 index 897b74822e8..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input_typearray.cwl +++ /dev/null @@ -1,42 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: input_typearray - cwlVersion: v1.0 - class: CommandLineTool - baseCommand: ['/bin/echo'] - stdout: "response.txt" - requirements: - - class: DockerRequirement - dockerPull: "ubuntu:latest" - - class: InlineJavascriptRequirement - arguments: - - position: 3 - valueFrom: "sentinel" - inputs: - value_f: - type: - - string - - File - inputBinding: - position: 1 - doc: "an input to test with a File value" - value_s: - type: - - string - - File - inputBinding: - position: 2 - doc: "an input to test with a string value" - outputs: - response_f: - type: string - outputBinding: - glob: response.txt - loadContents: true - outputEval: $(self[0].contents.split(" ")[0].split("/").slice(-1)[0]) - response_s: - type: string - outputBinding: - glob: response.txt - loadContents: true - outputEval: $(self[0].contents.split(" ")[1].split("/").slice(-1)[0]) diff --git a/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input_typearray.yml b/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input_typearray.yml deleted file mode 100644 index 421402d9289..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_input_typearray/input_typearray.yml +++ /dev/null @@ -1,4 +0,0 @@ -value_f: - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_input_typearray/input.txt" -value_s: "centaur/src/main/resources/standardTestCases/cwl_input_typearray/nonexistent_path.txt" diff --git a/centaur/src/main/resources/standardTestCases/cwl_inputdir_zero_doesnt_localize_papiv2.test b/centaur/src/main/resources/standardTestCases/cwl_inputdir_zero_doesnt_localize_papiv2.test deleted file mode 100644 index 564633779a7..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_inputdir_zero_doesnt_localize_papiv2.test +++ /dev/null @@ -1,17 +0,0 @@ -name: cwl_inputdir_zero_doesnt_localize_papiv2 -testFormat: workflowsuccess -backends: [Papiv2] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_resources/cwl_inputdir_zero_doesnt_localize.cwl -} - -metadata { - status: Succeeded - "outputs.inputdir_zero_doesnt_localize.errors": "" -} - -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: inputdir_zero_doesnt_localize diff --git a/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings.test b/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings.test deleted file mode 100644 index 89399e2e8fc..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings.test +++ /dev/null @@ -1,17 +0,0 @@ -name: cwl_interpolated_strings -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: interpolatedStrings -skipDescribeEndpointValidation: true - -files { - workflow: cwl_interpolated_strings/cwl_interpolated_strings.cwl - inputs: cwl_interpolated_strings/cwl_interpolated_strings.json -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.interpolatedStrings.rfc3092": "foo bar baz qux quux" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings/cwl_interpolated_strings.cwl b/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings/cwl_interpolated_strings.cwl deleted file mode 100644 index e650061a5cc..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings/cwl_interpolated_strings.cwl +++ /dev/null @@ -1,25 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: interpolatedStrings - class: CommandLineTool - cwlVersion: v1.0 - requirements: - - class: ShellCommandRequirement - hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" - - inputs: - - id: bar - type: string - - id: qux - type: string - - outputs: - - id: rfc3092 - type: string - - arguments: - - valueFrom: > - echo '{ "rfc3092": "foo $(inputs.bar) baz $(inputs.qux) quux" }' > cwl.output.json - shellQuote: false diff --git a/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings/cwl_interpolated_strings.json b/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings/cwl_interpolated_strings.json deleted file mode 100644 index 3f1bcedd11f..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_interpolated_strings/cwl_interpolated_strings.json +++ /dev/null @@ -1 +0,0 @@ -{"bar": "bar", "qux": "qux"} diff --git a/centaur/src/main/resources/standardTestCases/cwl_optionals.test b/centaur/src/main/resources/standardTestCases/cwl_optionals.test deleted file mode 100644 index 2d945f36ee0..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_optionals.test +++ /dev/null @@ -1,13 +0,0 @@ -name: cwl_optionals -testFormat: workflowsuccess -skipDescribeEndpointValidation: true - -files { - workflow: cwl_optionals/cwl_optionals.cwl - inputs: cwl_optionals/cwl_optionals.json -} - -metadata { - "actualWorkflowLanguage": CWL - "actualWorkflowLanguageVersion": v1.0 -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_optionals/cwl_optionals.cwl b/centaur/src/main/resources/standardTestCases/cwl_optionals/cwl_optionals.cwl deleted file mode 100755 index 592227bf025..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_optionals/cwl_optionals.cwl +++ /dev/null @@ -1,12 +0,0 @@ -cwlVersion: v1.0 -class: CommandLineTool -hints: - DockerRequirement: - dockerPull: "ubuntu:latest" -baseCommand: echo -inputs: - message: - type: string[]? - unsupplied_optional: - type: string[]? -outputs: [] diff --git a/centaur/src/main/resources/standardTestCases/cwl_optionals/cwl_optionals.json b/centaur/src/main/resources/standardTestCases/cwl_optionals/cwl_optionals.json deleted file mode 100644 index c76e98d49c3..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_optionals/cwl_optionals.json +++ /dev/null @@ -1 +0,0 @@ -{"message": ["foo","bar"]} diff --git a/centaur/src/main/resources/standardTestCases/cwl_output_json/cwl_output_json.cwl b/centaur/src/main/resources/standardTestCases/cwl_output_json/cwl_output_json.cwl deleted file mode 100644 index 406301e7ecb..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_output_json/cwl_output_json.cwl +++ /dev/null @@ -1,34 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: outputJson - # slightly modified version of conformance test #2 to remove the default File that doesn't work on PAPI - # It is instead being passed as an input through the input json file - class: CommandLineTool - cwlVersion: v1.0 - hints: - - class: DockerRequirement - dockerPull: python:2-slim - inputs: - - id: reference - type: File - inputBinding: { position: 2 } - - - id: reads - type: - type: array - items: File - inputBinding: { prefix: "-YYY" } - inputBinding: { position: 3, prefix: "-XXX" } - - - id: "args.py" - type: File - inputBinding: - position: -1 - - outputs: - # note the absence of any sort of valueFrom. - # The output value is generated from the "cwl.output.json" file created by the python script - args: string[] - - baseCommand: python - arguments: ["bwa", "mem"] diff --git a/centaur/src/main/resources/standardTestCases/cwl_output_json/cwl_output_json.inputs b/centaur/src/main/resources/standardTestCases/cwl_output_json/cwl_output_json.inputs deleted file mode 100644 index d83a30872a9..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_output_json/cwl_output_json.inputs +++ /dev/null @@ -1,22 +0,0 @@ -{ - "args.py": { - "class": "File", - "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/args.py" - }, - "reference": { - "class": "File", - "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/chr20.fa", - "size": 123, - "checksum": "sha1$hash" - }, - "reads": [ - { - "class": "File", - "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/example_human_Illumina.pe_1.fastq" - }, - { - "class": "File", - "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/example_human_Illumina.pe_2.fastq" - } - ] -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_output_json_jes.test b/centaur/src/main/resources/standardTestCases/cwl_output_json_jes.test deleted file mode 100644 index c7ddf584bd4..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_output_json_jes.test +++ /dev/null @@ -1,25 +0,0 @@ -name: cwl_output_json_jes -testFormat: workflowsuccess -backends: [Papi] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_output_json/cwl_output_json.cwl - inputs: cwl_output_json/cwl_output_json.inputs -} - -metadata { - status: Succeeded - "outputs.outputJson.args.0": "bwa" - "outputs.outputJson.args.1": "mem" - "outputs.outputJson.args.2": "chr20.fa" - "outputs.outputJson.args.3": "-XXX" - "outputs.outputJson.args.4": "-YYY" - "outputs.outputJson.args.5": "example_human_Illumina.pe_1.fastq" - "outputs.outputJson.args.6": "-YYY" - "outputs.outputJson.args.7": "example_human_Illumina.pe_2.fastq" -} - -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: outputJson diff --git a/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array.test b/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array.test deleted file mode 100644 index cea66a37026..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array.test +++ /dev/null @@ -1,17 +0,0 @@ -name: cwl_prefix_for_array -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: prefix-for-array -skipDescribeEndpointValidation: true - -files { - workflow: cwl_prefix_for_array/prefix_for_array.cwl - inputs: cwl_prefix_for_array/prefix_for_array.yml -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.prefix-for-array.out": "--bonus first second\n" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array/prefix_for_array.cwl b/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array/prefix_for_array.cwl deleted file mode 100644 index ca9044e5c61..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array/prefix_for_array.cwl +++ /dev/null @@ -1,23 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: prefix-for-array - cwlVersion: v1.0 - class: CommandLineTool - baseCommand: ['/bin/echo'] - stdout: "hello.txt" - requirements: - - class: DockerRequirement - dockerPull: "ubuntu:latest" - inputs: - bonus: - type: string[] - inputBinding: - prefix: "--bonus" - outputs: - out: - type: string - outputBinding: - glob: hello.txt - loadContents: true - outputEval: $(self[0].contents) - diff --git a/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array/prefix_for_array.yml b/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array/prefix_for_array.yml deleted file mode 100644 index 6a3badb824c..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_prefix_for_array/prefix_for_array.yml +++ /dev/null @@ -1 +0,0 @@ -bonus: ["first", "second"] diff --git a/centaur/src/main/resources/standardTestCases/cwl_recursive_link_directories.test b/centaur/src/main/resources/standardTestCases/cwl_recursive_link_directories.test deleted file mode 100644 index 390c1d66409..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_recursive_link_directories.test +++ /dev/null @@ -1,18 +0,0 @@ -name: cwl_recursive_link_directories -testFormat: workflowsuccess -backends: [Local] -backendsMode: only -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_recursive_link_directories/cwl_recursive_link_directories.cwl -} - -metadata { - status: Succeeded -} - -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: cwl_recursive_link_directories diff --git a/centaur/src/main/resources/standardTestCases/cwl_recursive_link_directories/cwl_recursive_link_directories.cwl b/centaur/src/main/resources/standardTestCases/cwl_recursive_link_directories/cwl_recursive_link_directories.cwl deleted file mode 100644 index a4a19fd6a5d..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_recursive_link_directories/cwl_recursive_link_directories.cwl +++ /dev/null @@ -1,19 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: cwl_recursive_link_directories - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - cwlVersion: v1.0 - hints: - - class: DockerRequirement - dockerPull: ubuntu:latest - inputs: [] - outputs: - output_dir: - type: Directory - outputBinding: - glob: work_dir - arguments: - - shellQuote: false - valueFrom: "mkdir work_dir && ln -s .. work_dir/link" diff --git a/centaur/src/main/resources/standardTestCases/cwl_relative_imports/cwl_glob_sort.cwl b/centaur/src/main/resources/standardTestCases/cwl_relative_imports/cwl_glob_sort.cwl deleted file mode 100644 index 330c896eea6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_relative_imports/cwl_glob_sort.cwl +++ /dev/null @@ -1,16 +0,0 @@ -cwlVersion: v1.0 -class: CommandLineTool -requirements: - - class: InlineJavascriptRequirement -hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" -inputs: [] -baseCommand: [touch, z, y, x, w, c, b, a] -outputs: - letters: - type: string - outputBinding: - glob: '?' - outputEval: | - ${ return self.sort(function(a,b) { return a.location > b.location ? 1 : (a.location < b.location ? -1 : 0) }).map(f => f.basename).join(" ") } diff --git a/centaur/src/main/resources/standardTestCases/cwl_relative_imports/workflow.cwl b/centaur/src/main/resources/standardTestCases/cwl_relative_imports/workflow.cwl deleted file mode 100644 index 0ba8fc7236d..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_relative_imports/workflow.cwl +++ /dev/null @@ -1,16 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: relative_imports - class: Workflow - - inputs: [] - outputs: - letters: - type: string - outputSource: globSort/letters - - steps: - globSort: - in: [] - run: cwl_glob_sort.cwl - out: [letters] diff --git a/centaur/src/main/resources/standardTestCases/cwl_relative_imports_url.test b/centaur/src/main/resources/standardTestCases/cwl_relative_imports_url.test deleted file mode 100644 index eabd10754f6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_relative_imports_url.test +++ /dev/null @@ -1,20 +0,0 @@ -name: cwl_relative_imports_url -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: relative_imports -skipDescribeEndpointValidation: true -# This test has experienced sporadic failures, and CWL is not fully supported. Therefore the test is being disabled. -ignore: true - -files { - workflowUrl: "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/centaur/src/main/resources/standardTestCases/cwl_relative_imports/workflow.cwl" -} - -metadata { - status: Succeeded - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "calls.relative_imports.globSort.outputs.letters": "a b c w x y z" - "outputs.relative_imports.letters": "a b c w x y z" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_relative_imports_zip.test b/centaur/src/main/resources/standardTestCases/cwl_relative_imports_zip.test deleted file mode 100644 index deccb8d73a3..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_relative_imports_zip.test +++ /dev/null @@ -1,22 +0,0 @@ -name: cwl_relative_imports_zip -testFormat: workflowsuccess -skipDescribeEndpointValidation: true - -files { - workflow: cwl_relative_imports/workflow.cwl - imports: [ - cwl_relative_imports/cwl_glob_sort.cwl - ] -} - -metadata { - status: Succeeded - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "calls.relative_imports.globSort.outputs.letters": "a b c w x y z" - "outputs.relative_imports.letters": "a b c w x y z" -} - -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: relative_imports diff --git a/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_disk_resources.cwl b/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_disk_resources.cwl deleted file mode 100644 index 683dd5b17e6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_disk_resources.cwl +++ /dev/null @@ -1,38 +0,0 @@ -cwlVersion: v1.0 -# Asks for 10 GB (~=10240 MB) of output dir, tmp dir and input dir. -# Unfortunately the internal metadata from the VM, despite having some information about the disks doesn't give their size. -# To avoid possibly unreliable computations based on free or other tool, instead use the GCE API directly to look up the disk info. -# To do that we need project name, zone and name of the disk which we all obtain via the internal metadata -# Of importance is the assumption that == -1 -# Note the "-1" after the instance name. That is because the disk just named is the boot disk. The working disk has a "-1" suffix -# The test expectation ensures that the output is 30, which means all 3 resources have been taken into account. -$namespaces: - dx: https://www.dnanexus.com/cwl# -$graph: -- id: diskSizeTool - class: CommandLineTool - cwlVersion: v1.0 - doc: "Asks for disk minimums" - requirements: - ResourceRequirement: - outdirMin: 10240 - tmpdirMin: 10240 - hints: - - class: ShellCommandRequirement - - class: DockerRequirement - dockerPull: gcr.io/google.com/cloudsdktool/cloud-sdk:slim - - class: dx:InputResourceRequirement - indirMin: 10240 - inputs: [] - outputs: - disk_size: - type: int - outputBinding: - glob: stdout - loadContents: true - outputEval: $(parseInt(self[0].contents.trim())) - arguments: - - valueFrom: > - apt-get install --assume-yes jq > /dev/null && NAME=`curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/name` && ZONE=`basename \`curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/instance/zone\`` && PROJECT=`curl -s -H "Metadata-Flavor: Google" http://metadata.google.internal/computeMetadata/v1/project/project-id` && curl -s -H "Authorization: Bearer `gcloud auth print-access-token`" "https://www.googleapis.com/compute/v1/projects/${PROJECT}/zones/${ZONE}/disks/${NAME}-1?fields=sizeGb" | jq -r '.sizeGb' - shellQuote: false - stdout: stdout diff --git a/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_inputdir_zero_doesnt_localize.cwl b/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_inputdir_zero_doesnt_localize.cwl deleted file mode 100644 index d874ee0bffb..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_inputdir_zero_doesnt_localize.cwl +++ /dev/null @@ -1,129 +0,0 @@ -cwlVersion: v1.0 -$namespaces: - dx: https://www.dnanexus.com/cwl# -$graph: -- id: inputdir_zero_doesnt_localize - class: Workflow - hints: - - class: ShellCommandRequirement - - class: DockerRequirement - dockerPull: gcr.io/google.com/cloudsdktool/cloud-sdk:slim - - class: dx:InputResourceRequirement - indirMin: 0 - inputs: [] - outputs: - - id: errors - type: File - outputSource: findFile/errors - steps: - - id: echo - in: [] - out: - - id: echoOut - - id: echoArrayOut - - id: echoRecordOut - - id: echoMaybeOut - run: - class: CommandLineTool - inputs: [] - outputs: - - id: echoOut - outputBinding: - glob: lonely_file - type: File - secondaryFiles: [.also] - - id: echoArrayOut - outputBinding: - glob: '*.txt*' - outputEval: $([self[0]]) - type: - items: File - type: array - - id: echoRecordOut - secondaryFiles: [.also] - type: - type: record - name: foo - fields: - - name: a - type: File - outputBinding: - glob: file_in_object - - id: echoMaybeOut - type: - - File - - 'null' - secondaryFiles: [.also] - outputBinding: - glob: maybe_file - arguments: - - valueFrom: | - echo "lonely_file" > lonely_file - echo "lonely_file.also" > lonely_file.also - echo "file_in_array.txt" > file_in_array.txt - echo "file_in_array.txt.also" > file_in_array.txt.also - echo "file_in_object" > file_in_object - echo "file_in_object.also" > file_in_object.also - echo "maybe_file" > maybe_file - echo "maybe_file.also" > maybe_file.also - shellQuote: false - - id: findFile - in: - - id: f - source: "#inputdir_zero_doesnt_localize/echo/echoOut" - - id: g - source: "#inputdir_zero_doesnt_localize/echo/echoArrayOut" - - id: h - source: "#inputdir_zero_doesnt_localize/echo/echoRecordOut" - - id: i - source: "#inputdir_zero_doesnt_localize/echo/echoMaybeOut" - out: - - id: errors - run: - inputs: - - id: f - type: File - secondaryFiles: [.also] - - id: i - type: File? - secondaryFiles: [.also] - - id: g - type: - type: array - items: File - secondaryFiles: [.also] - - id: h - secondaryFiles: [.also] - type: - type: record - name: foo - fields: - - name: a - type: File - outputs: - - id: errors - type: string - outputBinding: - glob: errors.txt - loadContents: true - outputEval: $(self[0].contents) - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - - class: InlineJavascriptRequirement - arguments: - # Unfortunately we can't use the same trick as in draft3_nio_file_papi1 because in papi 2 the instance metadata doesn't seem to contain - # the PAPI operation ID. So instead simply check that the input files are not there. This is not as neat as the PAPI1 version - # but should check the right thing as long as the files are localized (in general, but not here) somewhere in the same directory as where the command - # runs. If that changes then the directory where "find" searches should be updated - - valueFrom: | - touch errors.txt - find . -name lonely_file >> errors.txt - find . -name lonely_file.also >> errors.txt - find . -name file_in_array >> errors.txt - find . -name file_in_array.also >> errors.txt - find . -name file_in_object >> errors.txt - find . -name file_in_object.also >> errors.txt - find . -name maybe_file >> errors.txt - find . -name maybe_file.also >> errors.txt - shellQuote: false diff --git a/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_resources.cwl b/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_resources.cwl deleted file mode 100644 index 295302acc2e..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_resources/cwl_resources.cwl +++ /dev/null @@ -1,27 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: machineTypeTool - class: CommandLineTool - cwlVersion: v1.0 - doc: "Asks for CPU and Memory minimums" - requirements: - ResourceRequirement: - coresMin: 2 - coresMax: 2 - ramMin: 7GB - ramMax: 7GB - hints: - DockerRequirement: - dockerPull: python:latest - inputs: [] - outputs: - machine_type: - type: string - outputBinding: - glob: stdout - loadContents: true - outputEval: $(self[0].contents.trim()) - baseCommand: ['curl', 'http://metadata.google.internal/computeMetadata/v1/instance/machine-type', '-H', 'Metadata-Flavor: Google'] - stdout: stdout - - diff --git a/centaur/src/main/resources/standardTestCases/cwl_resources_papiv2.test b/centaur/src/main/resources/standardTestCases/cwl_resources_papiv2.test deleted file mode 100644 index ccb4f0ad534..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_resources_papiv2.test +++ /dev/null @@ -1,17 +0,0 @@ -name: cwl_resources_papiv2 -testFormat: workflowsuccess -backends: [Papiv2] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_resources/cwl_resources.cwl -} - -metadata { - status: Succeeded - "outputs.machineTypeTool.machine_type": "projects/1005074806481/machineTypes/custom-2-7168" -} - -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: machineTypeTool diff --git a/centaur/src/main/resources/standardTestCases/cwl_restart/cwl_restart.cwl b/centaur/src/main/resources/standardTestCases/cwl_restart/cwl_restart.cwl deleted file mode 100644 index 82664f739b1..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_restart/cwl_restart.cwl +++ /dev/null @@ -1,117 +0,0 @@ -cwlVersion: v1.0 -class: Workflow -id: cwl_restart -# Workflow-level DockerRequirement -requirements: - DockerRequirement: - dockerPull: "ubuntu:latest" -inputs: [] -outputs: -- id: flag - outputSource: "#third_step/flag" - type: boolean -steps: -- id: first_step - in: [] - out: - - id: firstfoofile - # A dash is not valid in a WDL identifier and would have broken older `IdentifierAndPathPattern` parsing. - - id: first-bar-file - run: - inputs: [] - outputs: - - id: firstfoofile - outputBinding: - glob: out.txt - type: File - - id: first-bar-file - outputBinding: - glob: out.txt - type: File - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - arguments: - - valueFrom: echo - - valueFrom: "I have a bad feeling about this" - - valueFrom: ">" - shellQuote: false - - valueFrom: "out.txt" -- id: cromwell_killer - in: - - id: fooinput - source: "#first_step/firstfoofile" - - id: bar-input - source: "#first_step/first-bar-file" - out: - - id: "footxt" - - id: "bar-txt" - run: - inputs: - - id: fooinput - type: File - - id: bar-input - type: File - outputs: - - id: "footxt" - outputBinding: - glob: foo.txt - type: File - - id: "bar-txt" - outputBinding: - glob: bar.txt - type: File - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - arguments: - - valueFrom: sleep - - valueFrom: "60" - - valueFrom: "&&" - shellQuote: false - - valueFrom: echo - - valueFrom: foo - - valueFrom: ">" - shellQuote: false - - valueFrom: "foo.txt" - - valueFrom: "&&" - shellQuote: false - - valueFrom: echo - - valueFrom: bar - - valueFrom: ">" - shellQuote: false - - valueFrom: "bar.txt" -- id: third_step - in: - - id: foo - source: "#cromwell_killer/footxt" - - id: bar - source: "#cromwell_killer/bar-txt" - out: - - id: flag - run: - inputs: - - id: foo - type: File - - id: bar - type: File - outputs: - - id: flag - outputBinding: - outputEval: $(true) - type: boolean - class: CommandLineTool - requirements: - - class: InlineJavascriptRequirement - - class: ShellCommandRequirement - arguments: - - valueFrom: echo - - valueFrom: "Are we alive???" - - valueFrom: "&&" - shellQuote: false - - valueFrom: echo - - valueFrom: $(inputs.foo) - - valueFrom: "&&" - shellQuote: false - - valueFrom: echo - - valueFrom: $(inputs.bar) diff --git a/centaur/src/main/resources/standardTestCases/cwl_restart_local_with_recover.test b/centaur/src/main/resources/standardTestCases/cwl_restart_local_with_recover.test deleted file mode 100644 index a45e97769cd..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_restart_local_with_recover.test +++ /dev/null @@ -1,21 +0,0 @@ -name: cwl_restart_local_with_recover -testFormat: CromwellRestartWithRecover -callMark: cromwell_killer -backendsMode: "only" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_restart/cwl_restart.cwl -} - -metadata { - status: Succeeded - "calls.first_step.executionStatus": Done - "calls.cromwell_killer.executionStatus": Done - "calls.third_step.executionStatus": Done -} - -workflowType: CWL -workflowTypeVersion: v1.0 diff --git a/centaur/src/main/resources/standardTestCases/cwl_run/1st-tool.cwl b/centaur/src/main/resources/standardTestCases/cwl_run/1st-tool.cwl deleted file mode 100644 index af0c4de297d..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_run/1st-tool.cwl +++ /dev/null @@ -1,9 +0,0 @@ -cwlVersion: v1.0 -class: CommandLineTool -baseCommand: echo -inputs: - message: - type: string - inputBinding: - position: 1 -outputs: [] diff --git a/centaur/src/main/resources/standardTestCases/cwl_run/echo-job.yml b/centaur/src/main/resources/standardTestCases/cwl_run/echo-job.yml deleted file mode 100644 index 913ff438103..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_run/echo-job.yml +++ /dev/null @@ -1 +0,0 @@ -message: Hello CWL! diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files.test b/centaur/src/main/resources/standardTestCases/cwl_secondary_files.test deleted file mode 100644 index 2c8f861e1e9..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files.test +++ /dev/null @@ -1,21 +0,0 @@ -name: cwl_secondary_files -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: main -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_secondary_files/cwl_secondary_files.cwl - inputs: cwl_secondary_files/cwl_secondary_files.yaml - options: cwl_secondary_files/cwl_secondary_files.options -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.main.the_answer": "$(42)" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bam.txt b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bam.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bam.txt.also b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bam.txt.also deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bar.txt b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bar.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bar.txt.also b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bar.txt.also deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/baz.txt b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/baz.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/baz.txt.also b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/baz.txt.also deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bim.txt b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bim.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bim.txt.also b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/bim.txt.also deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.cwl b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.cwl deleted file mode 100644 index 123a7b574a1..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.cwl +++ /dev/null @@ -1,44 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: main - class: CommandLineTool - hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" - inputs: - - id: command - type: string - - id: f - type: File - inputBinding: - position: 2 - secondaryFiles: [.also] - - id: of - type: File? - inputBinding: - position: 3 - secondaryFiles: [.also] - - id: fs - type: - type: array - items: File - inputBinding: - position: 4 - secondaryFiles: [.also] - - id: fr - secondaryFiles: [.also] - type: - type: record - name: foo - fields: - - name: a - type: File - inputBinding: - position: 5 - outputs: - the_answer: - type: string - outputBinding: - outputEval: ${ return "$(" + 42 + ")"; } - baseCommand: [] - arguments: ["bash", "-c", $(inputs.command)] diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.options b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.options deleted file mode 100644 index 71d245d2bf6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.options +++ /dev/null @@ -1,3 +0,0 @@ -{ - "backend": "Local" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.yaml b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.yaml deleted file mode 100644 index 928cfb8cbb3..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/cwl_secondary_files.yaml +++ /dev/null @@ -1,17 +0,0 @@ -# This command will stat the secondaryFiles and if they're missing bad times -command: stat $(echo $* | sed 's/.txt/.txt.also/g') > /dev/null -f: - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/foo.txt" -of: - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/bim.txt" -fs: - - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/bar.txt" - - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/baz.txt" -fr: - a: - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/bam.txt" \ No newline at end of file diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/foo.txt b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/foo.txt deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files/foo.txt.also b/centaur/src/main/resources/standardTestCases/cwl_secondary_files/foo.txt.also deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow.test b/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow.test deleted file mode 100644 index f6dff3fb727..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow.test +++ /dev/null @@ -1,21 +0,0 @@ -name: cwl_secondary_files_workflow -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: cwl_secondary_files_workflow -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_secondary_files_workflow/cwl_secondary_files_workflow.cwl - inputs: cwl_secondary_files_workflow/cwl_secondary_files_workflow.yaml - options: cwl_secondary_files_workflow/cwl_secondary_files_workflow.options -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "outputs.cwl_secondary_files_workflow.the_answer": "$(42)" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.cwl b/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.cwl deleted file mode 100644 index d5f9dc3f8c9..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.cwl +++ /dev/null @@ -1,55 +0,0 @@ -cwlVersion: v1.0 -$graph: -# The inputs of the workflow specify secondary files but NOT of the tool -# Verify that they are still propagated through -- id: cwl_secondary_files_workflow - class: Workflow - inputs: - - id: command - type: string - - id: wf_file_input - type: File - secondaryFiles: [.also] - - id: wf_file_input_array - type: - type: array - items: File - secondaryFiles: [.also] - outputs: - - id: the_answer - type: string - outputSource: run_tool/the_answer - steps: - - id: run_tool - run: "#cwl_secondary_files_workflow_tool" - in: - command: command - f: wf_file_input - fs: wf_file_input_array - out: - - id: the_answer -- id: cwl_secondary_files_workflow_tool - class: CommandLineTool - hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" - inputs: - - id: command - type: string - - id: f - type: File - inputBinding: - position: 2 - - id: fs - type: - type: array - items: File - inputBinding: - position: 3 - outputs: - the_answer: - type: string - outputBinding: - outputEval: ${ return "$(" + 42 + ")"; } - baseCommand: [] - arguments: ["bash", "-c", $(inputs.command)] diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.options b/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.options deleted file mode 100644 index 71d245d2bf6..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.options +++ /dev/null @@ -1,3 +0,0 @@ -{ - "backend": "Local" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.yaml b/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.yaml deleted file mode 100644 index e0cf43d4b1a..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_secondary_files_workflow/cwl_secondary_files_workflow.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# This command will stat the secondaryFiles and if they're missing bad times -command: stat $(echo $* | sed 's/.txt/.txt.also/g') > /dev/null -wf_file_input: - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/foo.txt" -wf_file_input_array: - - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/bar.txt" - - class: File - path: "centaur/src/main/resources/standardTestCases/cwl_secondary_files/baz.txt" diff --git a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression/cwl_stdout_expression.cwl b/centaur/src/main/resources/standardTestCases/cwl_stdout_expression/cwl_stdout_expression.cwl deleted file mode 100644 index b49ec9b1134..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression/cwl_stdout_expression.cwl +++ /dev/null @@ -1,17 +0,0 @@ -cwlVersion: v1.0 -$graph: -- id: cwl_stdout_expression - class: CommandLineTool - hints: - DockerRequirement: - dockerPull: "debian:stretch-slim" - inputs: - - id: foo - type: string - - id: bar - type: string - outputs: - b: stdout - stdout: "stdout-$(inputs.foo)-$(inputs.bar).txt" - baseCommand: [] - arguments: [echo, $(inputs.foo), $(inputs.bar)] diff --git a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression/cwl_stdout_expression.json b/centaur/src/main/resources/standardTestCases/cwl_stdout_expression/cwl_stdout_expression.json deleted file mode 100644 index 82e0295d318..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression/cwl_stdout_expression.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "foo": "baz", - "bar": "qux" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression_local.test b/centaur/src/main/resources/standardTestCases/cwl_stdout_expression_local.test deleted file mode 100644 index 13bfbb68e24..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression_local.test +++ /dev/null @@ -1,21 +0,0 @@ -name: cwl_stdout_expression_local -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: cwl_stdout_expression -backendsMode: "any" -backends: [Local, LocalNoDocker] -tags: [localdockertest] - -skipDescribeEndpointValidation: true - -files { - workflow: cwl_stdout_expression/cwl_stdout_expression.cwl - inputs: cwl_stdout_expression/cwl_stdout_expression.json -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "calls.cwl_stdout_expression.stdout": "<>/call-cwl_stdout_expression/execution/stdout-baz-qux.txt" -} diff --git a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression_papi.test b/centaur/src/main/resources/standardTestCases/cwl_stdout_expression_papi.test deleted file mode 100644 index b48a0effa79..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwl_stdout_expression_papi.test +++ /dev/null @@ -1,18 +0,0 @@ -name: cwl_stdout_expression_papi -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: cwl_stdout_expression -backends: [Papi2] -skipDescribeEndpointValidation: true - -files { - workflow: cwl_stdout_expression/cwl_stdout_expression.cwl - inputs: cwl_stdout_expression/cwl_stdout_expression.json -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "calls.cwl_stdout_expression.stdout": "<>call-cwl_stdout_expression/stdout-baz-qux.txt" -} diff --git a/centaur/src/main/resources/standardTestCases/cwls/scatter-job1.json b/centaur/src/main/resources/standardTestCases/cwls/scatter-job1.json deleted file mode 100644 index b3ab2751243..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwls/scatter-job1.json +++ /dev/null @@ -1,4 +0,0 @@ -{ - "inp": ["one", "two", "three", "four"], - "inp2": "yeahhh" -} diff --git a/centaur/src/main/resources/standardTestCases/cwls/scatter-wf1.cwl b/centaur/src/main/resources/standardTestCases/cwls/scatter-wf1.cwl deleted file mode 100644 index dc46aa7da4a..00000000000 --- a/centaur/src/main/resources/standardTestCases/cwls/scatter-wf1.cwl +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env cwl-runner -cwlVersion: v1.0 -class: Workflow -inputs: - inp: string[] - inp2: string -outputs: - out: - type: string[] - outputSource: step1/echo_out - -requirements: - - class: ScatterFeatureRequirement - - class: DockerRequirement - dockerPull: ubuntu:latest - -steps: - step1: - in: - echo_in: inp - echo_in2: inp2 - out: [echo_out] - scatter: echo_in - run: - class: CommandLineTool - inputs: - echo_in: - type: string - inputBinding: {} - echo_in2: - type: string - inputBinding: {} - outputs: - echo_out: - type: string - outputBinding: - glob: "step1_out" - loadContents: true - outputEval: $(self[0].contents) - baseCommand: "echo" - arguments: - - "-n" - - "foo" - stdout: "step1_out" diff --git a/centaur/src/main/resources/standardTestCases/declarations.test b/centaur/src/main/resources/standardTestCases/declarations.test index 53766678b26..bae479145f0 100644 --- a/centaur/src/main/resources/standardTestCases/declarations.test +++ b/centaur/src/main/resources/standardTestCases/declarations.test @@ -2,7 +2,6 @@ name: declarations testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: declarations/declarations.wdl diff --git a/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test b/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test index d18f39677f6..ff1f4416a24 100644 --- a/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test +++ b/centaur/src/main/resources/standardTestCases/declarations_as_nodes.test @@ -1,6 +1,5 @@ name: declarations_as_nodes testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: declarations_as_nodes/declarations_as_nodes.wdl diff --git a/centaur/src/main/resources/standardTestCases/directory_type_local.test b/centaur/src/main/resources/standardTestCases/directory_type_local.test index f2db3398d96..ae0f90dcc72 100644 --- a/centaur/src/main/resources/standardTestCases/directory_type_local.test +++ b/centaur/src/main/resources/standardTestCases/directory_type_local.test @@ -1,11 +1,11 @@ name: directory_type_local testFormat: workflowsuccess -tags: [localdockertest, "wdl_biscayne"] +tags: [localdockertest, "wdl_cascades"] backends: [Local, LocalNoDocker] files { - workflow: wdl_biscayne/directory_type/directory_type.wdl - inputs: wdl_biscayne/directory_type/directory_type_local_inputs.json + workflow: wdl_cascades/directory_type/directory_type.wdl + inputs: wdl_cascades/directory_type/directory_type_local_inputs.json } metadata { diff --git a/centaur/src/main/resources/standardTestCases/directory_type_output_papi.test b/centaur/src/main/resources/standardTestCases/directory_type_output_papi.test index f66dfd656b3..67e435db619 100644 --- a/centaur/src/main/resources/standardTestCases/directory_type_output_papi.test +++ b/centaur/src/main/resources/standardTestCases/directory_type_output_papi.test @@ -1,10 +1,10 @@ name: directory_type_output_papi testFormat: workflowsuccess -tags: ["wdl_biscayne"] +tags: ["wdl_cascades"] backends: [Papi] files { - workflow: wdl_biscayne/directory_type_output/directory_type_output.wdl + workflow: wdl_cascades/directory_type_output/directory_type_output.wdl } metadata { diff --git a/centaur/src/main/resources/standardTestCases/directory_type_papi.test b/centaur/src/main/resources/standardTestCases/directory_type_papi.test index 988a609d237..4d953e03c4e 100644 --- a/centaur/src/main/resources/standardTestCases/directory_type_papi.test +++ b/centaur/src/main/resources/standardTestCases/directory_type_papi.test @@ -1,10 +1,10 @@ name: directory_type_papi testFormat: workflowsuccess -tags: ["wdl_biscayne"] +tags: ["wdl_cascades"] backends: [Papi] files { - workflow: wdl_biscayne/directory_type/directory_type.wdl + workflow: wdl_cascades/directory_type/directory_type.wdl } metadata { diff --git a/centaur/src/main/resources/standardTestCases/draft3_read_file_limits.test b/centaur/src/main/resources/standardTestCases/draft3_read_file_limits.test index 4a9af0c8813..4bcbdd38db7 100644 --- a/centaur/src/main/resources/standardTestCases/draft3_read_file_limits.test +++ b/centaur/src/main/resources/standardTestCases/draft3_read_file_limits.test @@ -2,6 +2,7 @@ name: draft3_read_file_limits testFormat: workflowfailure workflowType: WDL workflowTypeVersion: 1.0 +tags: [batchexclude] files { workflow: wdl_draft3/read_file_limits/read_file_limits.wdl diff --git a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.inputs b/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.inputs index d5738ac4d17..d1712641dd2 100644 --- a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.inputs +++ b/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.inputs @@ -1,5 +1,5 @@ { - # Martha does not return service accounts for JDR paths. Therefore they shouldn't need to be localized using the + # DRSHub does not return service accounts for JDR paths. Therefore they shouldn't need to be localized using the # Cromwell custom DOS/DRS localizer. # # However, the first file1 was generated before JDR stared saving file names at the end of the gsUri. diff --git a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.wdl b/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.wdl index ba2a17f292d..e9b56af98d2 100644 --- a/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.wdl +++ b/centaur/src/main/resources/standardTestCases/drs_tests/drs_usa_jdr.wdl @@ -61,7 +61,7 @@ task localize_jdr_drs_with_usa { } runtime { - docker: "ubuntu" + docker: "ubuntu:latest" backend: "papi-v2-usa" } } @@ -88,7 +88,7 @@ task skip_localize_jdr_drs_with_usa { } runtime { - docker: "ubuntu" + docker: "ubuntu:latest" backend: "papi-v2-usa" } } @@ -109,7 +109,7 @@ task read_drs_with_usa { } runtime { - docker: "ubuntu" + docker: "ubuntu:latest" backend: "papi-v2-usa" } } diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test b/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test index 4025860ed8d..510c9215c1f 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test +++ b/centaur/src/main/resources/standardTestCases/drs_usa_jdr.test @@ -1,6 +1,7 @@ name: drs_usa_jdr testFormat: WorkflowSuccess backends: ["papi-v2-usa"] +tags: [ drs ] skipDescribeEndpointValidation: true files { diff --git a/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test b/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test index ab050aea69f..15e41d95529 100644 --- a/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test +++ b/centaur/src/main/resources/standardTestCases/drs_usa_jdr_preresolve.test @@ -1,6 +1,7 @@ name: drs_usa_jdr_preresolve testFormat: WorkflowSuccess backends: ["papi-v2-usa"] +tags: [ drs ] skipDescribeEndpointValidation: true files { diff --git a/centaur/src/main/resources/standardTestCases/expressionLib.test b/centaur/src/main/resources/standardTestCases/expressionLib.test deleted file mode 100644 index 74299af9bc6..00000000000 --- a/centaur/src/main/resources/standardTestCases/expressionLib.test +++ /dev/null @@ -1,15 +0,0 @@ -name: expression_lib_cwl -cwlVersion: 1.0 -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: expressionLib/expressionLib.cwl -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 -} diff --git a/centaur/src/main/resources/standardTestCases/expressionLib/expressionLib.cwl b/centaur/src/main/resources/standardTestCases/expressionLib/expressionLib.cwl deleted file mode 100644 index fe8df12b4b0..00000000000 --- a/centaur/src/main/resources/standardTestCases/expressionLib/expressionLib.cwl +++ /dev/null @@ -1,14 +0,0 @@ -class: CommandLineTool -cwlVersion: v1.0 -hints: - DockerRequirement: - dockerPull: "ubuntu:latest" -requirements: - InlineJavascriptRequirement: - expressionLib: - - "function foo() { return 2; }" -inputs: [] -outputs: - out: stdout -arguments: [echo, $(foo())] -stdout: whatever.txt diff --git a/centaur/src/main/resources/standardTestCases/failing_return_code.test b/centaur/src/main/resources/standardTestCases/failing_return_code.test new file mode 100644 index 00000000000..163715222cf --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failing_return_code.test @@ -0,0 +1,11 @@ +name: failing_return_code +testFormat: workflowfailure + +files { + workflow: failing_return_code/failing_return_code.wdl +} + +metadata { + workflowName: FailingReturnCode + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl b/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl new file mode 100644 index 00000000000..f393698d2b7 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failing_return_code/failing_return_code.wdl @@ -0,0 +1,20 @@ +version development-1.1 + +workflow FailingReturnCode { + call FailingReturnCodeSet +} + +task FailingReturnCodeSet { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + returnCodes: [0, 5, 10] + docker: "ubuntu:latest" + } +} diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test index 1ac12322f4f..182a833fbdb 100644 --- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test +++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_jes.test @@ -2,6 +2,7 @@ name: failures.restart_while_failing_jes testFormat: WorkflowFailureRestartWithRecover callMark: restart_while_failing.B1 backends: [Papi] +tags: [restart] files { workflow: failures/restart_while_failing/restart_while_failing.wdl diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test index f686b16c817..58abbf78245 100644 --- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test +++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_local.test @@ -3,7 +3,7 @@ testFormat: WorkflowFailureRestartWithRecover callMark: restart_while_failing.B1 backendsMode: "only" backends: [Local, LocalNoDocker] -tags: [localdockertest] +tags: [localdockertest, restart] files { workflow: failures/restart_while_failing/restart_while_failing.wdl diff --git a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test index 64ea5fd8906..9bb9c1a55d6 100644 --- a/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test +++ b/centaur/src/main/resources/standardTestCases/failures.restart_while_failing_tes.test @@ -2,7 +2,7 @@ name: failures.restart_while_failing_tes testFormat: WorkflowFailureRestartWithoutRecover callMark: restart_while_failing.B1 backends: [TES] -tags: [localdockertest] +tags: [localdockertest, restart] files { workflow: failures/restart_while_failing/restart_while_failing.wdl diff --git a/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs new file mode 100644 index 00000000000..6f8cb64990a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs @@ -0,0 +1,4 @@ +{ + "MinimalStructExample.test": "Hello World", + "MinimalStructExample.integer": 2 +} diff --git a/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl new file mode 100644 index 00000000000..dce55da0bae --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl @@ -0,0 +1,27 @@ +version 1.0 + + +struct firstLayer { + String first + Int number +} + +struct secondLayer { + String second + firstLayer lowerLayer +} + + +workflow MinimalStructExample { + input { + String test + Int integer + } + + firstLayer example1 = {"first": test, "number": integer} + secondLayer example2 = {"second": test, "lowerLayer": example1} + + output { + String example3 = example2.second + } +} diff --git a/centaur/src/main/resources/standardTestCases/failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl b/centaur/src/main/resources/standardTestCases/failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl new file mode 100644 index 00000000000..733d448a5ed --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl @@ -0,0 +1,8 @@ +version 1.0 + +workflow break_with_stderr { + + output { + File load_data_csv = select_first([stdout(), stderr()]) + } +} diff --git a/centaur/src/main/resources/standardTestCases/hello.test b/centaur/src/main/resources/standardTestCases/hello.test index 33a49bd4071..2f7fd67087f 100644 --- a/centaur/src/main/resources/standardTestCases/hello.test +++ b/centaur/src/main/resources/standardTestCases/hello.test @@ -1,6 +1,5 @@ name: hello testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: hello/hello.wdl diff --git a/centaur/src/main/resources/standardTestCases/hello_cwl.test b/centaur/src/main/resources/standardTestCases/hello_cwl.test deleted file mode 100644 index 18712fd5404..00000000000 --- a/centaur/src/main/resources/standardTestCases/hello_cwl.test +++ /dev/null @@ -1,12 +0,0 @@ -name: hello_cwl -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -workflowRoot: hello -skipDescribeEndpointValidation: true - -files { - workflow: hello_cwl/hello.cwl - inputs: hello_cwl/hello.inputs - "outputs.hello.salutation": "Hello m'Lord!" -} diff --git a/centaur/src/main/resources/standardTestCases/hello_cwl/hello.cwl b/centaur/src/main/resources/standardTestCases/hello_cwl/hello.cwl deleted file mode 100644 index deb1e4d1f4a..00000000000 --- a/centaur/src/main/resources/standardTestCases/hello_cwl/hello.cwl +++ /dev/null @@ -1,21 +0,0 @@ -cwlVersion: v1.0 -$graph: - id: hello - class: CommandLineTool - requirements: - - class: DockerRequirement - dockerPull: ubuntu:latest - baseCommand: echo - inputs: - message: - type: string - inputBinding: - position: 1 - stdout: hello-stdout.txt - outputs: - - id: salutation - type: string - outputBinding: - glob: hello-stdout.txt - loadContents: true - outputEval: $(self[0].contents.trim()) diff --git a/centaur/src/main/resources/standardTestCases/hello_cwl/hello.inputs b/centaur/src/main/resources/standardTestCases/hello_cwl/hello.inputs deleted file mode 100644 index 11888b548ad..00000000000 --- a/centaur/src/main/resources/standardTestCases/hello_cwl/hello.inputs +++ /dev/null @@ -1,3 +0,0 @@ -# Seemingly innocuous but this particular message actually tests proper shell quoting of -# spaces, single quotes, and exclamation points. -message: Hello m'Lord! diff --git a/centaur/src/main/resources/standardTestCases/http_inputs/http_inputs.cwl b/centaur/src/main/resources/standardTestCases/http_inputs/http_inputs.cwl deleted file mode 100644 index e476be06d12..00000000000 --- a/centaur/src/main/resources/standardTestCases/http_inputs/http_inputs.cwl +++ /dev/null @@ -1,49 +0,0 @@ -cwlVersion: v1.0 -$graph: - - id: http_inputs - class: Workflow - inputs: - - id: jamie - type: File - outputs: - - id: md5 - outputSource: "#http_inputs/sum/md5" - type: int - steps: - - id: sum - in: - - id: jamie - source: "#http_inputs/jamie" - out: - - id: md5 - # Workflow step-level DockerRequirement - requirements: - DockerRequirement: - dockerPull: "ubuntu:latest" - run: - inputs: - - id: jamie - type: File - outputs: - - id: md5 - outputBinding: - glob: md5-stdOut.txt - loadContents: true - outputEval: $(self[0].contents) - type: string - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - - class: InlineJavascriptRequirement - arguments: - - valueFrom: "/usr/bin/md5sum" - shellQuote: false - - valueFrom: $(inputs.jamie) - shellQuote: true - - valueFrom: '|' - shellQuote: false - - valueFrom: cut - shellQuote: false - - valueFrom: -c1-32 - shellQuote: false - stdout: md5-stdOut.txt diff --git a/centaur/src/main/resources/standardTestCases/http_inputs/http_inputs_cwl.inputs b/centaur/src/main/resources/standardTestCases/http_inputs/http_inputs_cwl.inputs deleted file mode 100644 index 9a7137d1bf9..00000000000 --- a/centaur/src/main/resources/standardTestCases/http_inputs/http_inputs_cwl.inputs +++ /dev/null @@ -1,3 +0,0 @@ -{ - "jamie": "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/docs/jamie_the_cromwell_pig.png", -} diff --git a/centaur/src/main/resources/standardTestCases/http_inputs_cwl.test b/centaur/src/main/resources/standardTestCases/http_inputs_cwl.test deleted file mode 100644 index 7ed111e3ec7..00000000000 --- a/centaur/src/main/resources/standardTestCases/http_inputs_cwl.test +++ /dev/null @@ -1,15 +0,0 @@ -name: http_inputs_cwl -backends: [Local, Papiv2] -testFormat: workflowsuccess -workflowRoot: http_inputs -skipDescribeEndpointValidation: true - -files { - workflow: http_inputs/http_inputs.cwl - inputs: http_inputs/http_inputs_cwl.inputs -} - -metadata { - status: Succeeded - "outputs.http_inputs.md5": "602d0a4c1d54b7302e2d245a66a1b609\n" -} diff --git a/centaur/src/main/resources/standardTestCases/input_localization/localize_file_larger_than_disk_space.wdl b/centaur/src/main/resources/standardTestCases/input_localization/localize_file_larger_than_disk_space.wdl deleted file mode 100644 index b90c63fe928..00000000000 --- a/centaur/src/main/resources/standardTestCases/input_localization/localize_file_larger_than_disk_space.wdl +++ /dev/null @@ -1,27 +0,0 @@ -version 1.0 - -task localize_file { - input { - File input_file - } - command { - cat "localizing file over 1 GB" - } - runtime { - docker: "ubuntu:latest" - disks: "local-disk 1 HDD" - } - output { - String out = read_string(stdout()) - } -} - -workflow localize_file_larger_than_disk_space { - File wf_input = "gs://cromwell_test_bucket/file_over_1_gb.txt" - - call localize_file { input: input_file = wf_input } - - output { - String content = localize_file.out - } -} diff --git a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test new file mode 100644 index 00000000000..8f55af82262 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code.test @@ -0,0 +1,11 @@ +name: invalid_return_codes_and_continue_on_return_code +testFormat: workflowfailure + +files { + workflow: invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl +} + +metadata { + workflowName: InvalidReturnCodeAndContinueOnReturnCode + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl new file mode 100644 index 00000000000..ea2233ab355 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/invalid_return_codes_and_continue_on_return_code/invalid_return_codes_and_continue_on_return_code.wdl @@ -0,0 +1,21 @@ +version development-1.1 + +workflow InvalidReturnCodeAndContinueOnReturnCode { + call InvalidReturnCodeContinueOnReturnCode +} + +task InvalidReturnCodeContinueOnReturnCode { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [5, 10, 15] + continueOnReturnCode: [1] + } +} diff --git a/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_use_good_local.test b/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_use_good_local.test index 2d3bc8a4e31..4249e041c47 100644 --- a/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_use_good_local.test +++ b/centaur/src/main/resources/standardTestCases/invalidate_bad_caches_use_good_local.test @@ -3,6 +3,10 @@ testFormat: workflowsuccess backends: [Local] tags: [localdockertest] +# This test stopped working 8/23 but its cloud equivalent that we care about is fine [0] +# [0] `invalidate_bad_caches_use_good_jes.test` +ignore: true + files { workflow: invalidate_bad_caches/invalidate_bad_caches_use_good.wdl inputs: invalidate_bad_caches/local.inputs diff --git a/centaur/src/main/resources/standardTestCases/localize_file_larger_than_disk_space.test b/centaur/src/main/resources/standardTestCases/localize_file_larger_than_disk_space.test deleted file mode 100644 index bee99677b01..00000000000 --- a/centaur/src/main/resources/standardTestCases/localize_file_larger_than_disk_space.test +++ /dev/null @@ -1,17 +0,0 @@ -name: localize_file_larger_than_disk_space -testFormat: workflowfailure -backends: [Papiv2] -workflowType: WDL -workflowTypeVersion: 1.0 -tags: ["wdl_1.0"] - -files { - workflow: input_localization/localize_file_larger_than_disk_space.wdl -} - -metadata { - workflowName: localize_file_larger_than_disk_space - status: Failed - "failures.0.message": "Workflow failed" - "failures.0.causedBy.0.message": "Task localize_file_larger_than_disk_space.localize_file:NA:1 failed. The job was stopped before the command finished. PAPI error code 9. Please check the log file for more details: gs://cloud-cromwell-dev-self-cleaning/cromwell_execution/ci/localize_file_larger_than_disk_space/<>/call-localize_file/localize_file.log." -} diff --git a/centaur/src/main/resources/standardTestCases/long_cmd.test b/centaur/src/main/resources/standardTestCases/long_cmd.test index 40b6110b629..cef5fda2177 100644 --- a/centaur/src/main/resources/standardTestCases/long_cmd.test +++ b/centaur/src/main/resources/standardTestCases/long_cmd.test @@ -9,6 +9,7 @@ name: long_cmd testFormat: workflowsuccess +tags: [batchexclude] files { workflow: long_cmd/long_cmd.wdl diff --git a/centaur/src/main/resources/standardTestCases/nested_struct_bad_instantiation.test b/centaur/src/main/resources/standardTestCases/nested_struct_bad_instantiation.test new file mode 100644 index 00000000000..a2608a0262c --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/nested_struct_bad_instantiation.test @@ -0,0 +1,14 @@ +name: nested_struct_bad_instantiation +testFormat: workflowfailure + +files { + workflow: failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.wdl + inputs: failures/nested_struct_bad_instantiation/nested_struct_bad_instantiation.inputs +} + +metadata { + workflowName: MinimalStructExample + status: Failed + "failures.0.message": "Workflow failed" + "failures.0.causedBy.0.message": "Failed to evaluate 'example2' (reason 1 of 1): Evaluating { \"second\": test, \"lowerLayer\": example1 } failed: Cannot construct WomMapType(WomStringType,WomAnyType) with mixed types in map values: [WomString(Hello World), WomObject(Map(first -> WomString(Hello World), number -> WomInteger(2)),WomCompositeType(Map(first -> WomStringType, number -> WomIntegerType),Some(firstLayer)))]" +} diff --git a/centaur/src/main/resources/standardTestCases/object_access.test b/centaur/src/main/resources/standardTestCases/object_access.test index 1e824f6e78f..84d76d5a893 100644 --- a/centaur/src/main/resources/standardTestCases/object_access.test +++ b/centaur/src/main/resources/standardTestCases/object_access.test @@ -1,6 +1,5 @@ name: object_access testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: object_access/object_access.wdl diff --git a/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test b/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test index 5aca2634edb..7b38c3a25d7 100644 --- a/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test +++ b/centaur/src/main/resources/standardTestCases/papi_cpu_platform.test @@ -8,9 +8,10 @@ files { metadata { status: Succeeded - "outputs.cpus.cascadeLake.cpuPlatform": "Intel Cascade Lake" "outputs.cpus.broadwell.cpuPlatform": "Intel Broadwell" "outputs.cpus.haswell.cpuPlatform": "Intel Haswell" + "outputs.cpus.cascadeLake.cpuPlatform": "Intel Cascade Lake" + "outputs.cpus.iceLake.cpuPlatform": "Intel Ice Lake" "outputs.cpus.rome.cpuPlatform": "AMD Rome" } diff --git a/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl b/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl index 39d25e6fe47..3ec1bd77cae 100644 --- a/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl +++ b/centaur/src/main/resources/standardTestCases/papi_cpu_platform/papi_cpu_platform.wdl @@ -24,8 +24,9 @@ task cpu_platform { } workflow cpus { - call cpu_platform as haswell { input: cpu_platform = "Intel Haswell" } - call cpu_platform as broadwell { input: cpu_platform = "Intel Broadwell" } - call cpu_platform as cascadeLake { input: cpu_platform = "Intel Cascade Lake" } - call cpu_platform as rome {input: cpu_platform = "AMD Rome" } + call cpu_platform as haswell { input: cpu_platform = "Intel Haswell" } + call cpu_platform as broadwell { input: cpu_platform = "Intel Broadwell" } + call cpu_platform as cascadeLake { input: cpu_platform = "Intel Cascade Lake" } + call cpu_platform as iceLake { input: cpu_platform = "Intel Ice Lake" } + call cpu_platform as rome { input: cpu_platform = "AMD Rome" } } diff --git a/centaur/src/main/resources/standardTestCases/read_file_limits.test b/centaur/src/main/resources/standardTestCases/read_file_limits.test index 0079401812a..734ab809b92 100644 --- a/centaur/src/main/resources/standardTestCases/read_file_limits.test +++ b/centaur/src/main/resources/standardTestCases/read_file_limits.test @@ -1,5 +1,6 @@ name: read_file_limits testFormat: workflowfailure +tags: [batchexclude] files { workflow: read_file_limits/read_file_limits.wdl diff --git a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_mounted_only_if_requested.wdl b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_mounted_only_if_requested.wdl new file mode 100644 index 00000000000..33153c1157c --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_mounted_only_if_requested.wdl @@ -0,0 +1,49 @@ +version 1.0 + + +workflow ReferenceDiskMountedOnlyIfRequested { + call MentionsNirvanaReference {} + output { + Boolean disk_mounted = MentionsNirvanaReference.disk_mounted + } +} + + +task MentionsNirvanaReference { + input { + # Tiny 116 byte reference file, certainly not worth attaching a 55 GiB Nirvana reference disk for this. + File mention = + "gs://broad-public-datasets/gvs/vat-annotations/Nirvana/3.18.1/SupplementaryAnnotation/GRCh38/MITOMAP_20200819.nsa.idx" + } + command <<< + PS4='\D{+%F %T} \w $ ' + set -o nounset -o xtrace + + # Debug output + lsblk + + CANDIDATE_MOUNT_POINT=$(lsblk | sed -E -n 's!.*(/mnt/[a-f0-9]+).*!\1!p') + + if [[ ! -z ${CANDIDATE_MOUNT_POINT} ]]; then + echo "Found unexpected mounted disk, investigating further." + find ${CANDIDATE_MOUNT_POINT} -print | tee find.out + + if grep -i nirvana find.out; then + echo "Found what appears to be a Nirvana reference disk." + else + echo "Found unknown volume mounted, see 'find.out' for manifest." + fi + echo true > disk_mounted.out + else + echo false > disk_mounted.out + fi + >>> + runtime { + docker: "ubuntu:latest" + backend: "Papiv2-Reference-Disk-Localization" + } + output { + Boolean disk_mounted = read_boolean("disk_mounted.out") + File? find_out = "find.out" + } +} diff --git a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs index 8e28a8ef541..4caf94c0f77 100644 --- a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs +++ b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.inputs @@ -1,3 +1,5 @@ { - "wf_reference_disk_test.check_if_localized_as_symlink.reference_file_input": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.tile_db_header.vcf" + "wf_reference_disk_test.broad_reference_file_input": "gs://gcp-public-data--broad-references/hg19/v0/Homo_sapiens_assembly19.fasta.fai", + "wf_reference_disk_test.nirvana_reference_file_input": "gs://broad-public-datasets/gvs/vat-annotations/Nirvana/3.18.1/SupplementaryAnnotation/GRCh38/phyloP_hg38.npd.idx", + "wf_reference_disk_test.nirvana_reference_file_metachar_input": "gs://broad-public-datasets/gvs/vat-annotations/Nirvana/3.18.1/SupplementaryAnnotation/GRCh38/1000_Genomes_Project_(SV)_Phase_3_v5a.nsi" } diff --git a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.wdl b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.wdl index 43a298f18ab..51fb467598b 100644 --- a/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.wdl +++ b/centaur/src/main/resources/standardTestCases/reference_disk/reference_disk_test.wdl @@ -1,11 +1,38 @@ -task check_if_localized_as_symlink { - File reference_file_input - command { - # print true if file is a symlink, otherwise print false - if test -h ${reference_file_input}; then echo "true"; else echo "false"; fi; +version 1.0 + +task check_if_localized_with_valid_symlink { + input { + File broad_reference_file_input + File nirvana_reference_file_input + File nirvana_reference_file_metachar_input } + String broad_input_valid_symlink = "broad_input_valid_symlink.txt" + String nirvana_input_valid_symlink = "nirvana_input_valid_symlink.txt" + String nirvana_metachar_input_valid_symlink = "nirvana_metachar_input_valid_symlink.txt" + command <<< + PS4='\D{+%F %T} \w $ ' + set -o nounset -o pipefail -o xtrace + + # Echo true to stdout if the argument is a symlink pointing to an extant file, otherwise echo false. + check_if_valid_symlink() { + local reference_input="$1" + + if [[ -h "${reference_input}" && -f $(readlink "${reference_input}") ]]; then + echo true + else + echo false + fi + } + + check_if_valid_symlink "~{broad_reference_file_input}" > ~{broad_input_valid_symlink} + check_if_valid_symlink "~{nirvana_reference_file_input}" > ~{nirvana_input_valid_symlink} + check_if_valid_symlink "~{nirvana_reference_file_metachar_input}" > ~{nirvana_metachar_input_valid_symlink} + + >>> output { - Boolean is_symlink = read_boolean(stdout()) + Boolean is_broad_input_valid_symlink = read_boolean("~{broad_input_valid_symlink}") + Boolean is_nirvana_input_valid_symlink = read_boolean("~{nirvana_input_valid_symlink}") + Boolean is_nirvana_metachar_input_valid_symlink = read_boolean("~{nirvana_metachar_input_valid_symlink}") } runtime { docker: "ubuntu:latest" @@ -14,8 +41,20 @@ task check_if_localized_as_symlink { } workflow wf_reference_disk_test { - call check_if_localized_as_symlink + input { + File broad_reference_file_input + File nirvana_reference_file_input + File nirvana_reference_file_metachar_input + } + call check_if_localized_with_valid_symlink { + input: + broad_reference_file_input = broad_reference_file_input, + nirvana_reference_file_input = nirvana_reference_file_input, + nirvana_reference_file_metachar_input = nirvana_reference_file_metachar_input + } output { - Boolean is_input_file_a_symlink = check_if_localized_as_symlink.is_symlink + Boolean is_broad_input_file_a_valid_symlink = check_if_localized_with_valid_symlink.is_broad_input_valid_symlink + Boolean is_nirvana_input_file_a_valid_symlink = check_if_localized_with_valid_symlink.is_nirvana_input_valid_symlink + Boolean is_nirvana_metachar_input_file_a_valid_symlink = check_if_localized_with_valid_symlink.is_nirvana_metachar_input_valid_symlink } } diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_false_options.test b/centaur/src/main/resources/standardTestCases/reference_disk_false_options.test index b75ef58dabc..bb8be222691 100644 --- a/centaur/src/main/resources/standardTestCases/reference_disk_false_options.test +++ b/centaur/src/main/resources/standardTestCases/reference_disk_false_options.test @@ -11,5 +11,7 @@ files { metadata { workflowName: wf_reference_disk_test status: Succeeded - "outputs.wf_reference_disk_test.is_input_file_a_symlink": false + "outputs.wf_reference_disk_test.is_broad_input_file_a_valid_symlink": false + "outputs.wf_reference_disk_test.is_nirvana_input_file_a_valid_symlink": false + "outputs.wf_reference_disk_test.is_nirvana_metachar_input_file_a_valid_symlink": false } diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_false.test b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_false.test new file mode 100644 index 00000000000..32c6d83e5c3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_false.test @@ -0,0 +1,14 @@ +name: reference_disk_mounted_only_if_requested_false +testFormat: workflowsuccess +backends: [Papiv2-Reference-Disk-Localization] + +files { + workflow: reference_disk/reference_disk_mounted_only_if_requested.wdl + options: reference_disk/reference_disk_test_false.options.json +} + +metadata { + workflowName: ReferenceDiskMountedOnlyIfRequested + status: Succeeded + "outputs.ReferenceDiskMountedOnlyIfRequested.disk_mounted": false +} diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_true.test b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_true.test new file mode 100644 index 00000000000..165e1c6d1f4 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_true.test @@ -0,0 +1,14 @@ +name: reference_disk_mounted_only_if_requested_true +testFormat: workflowsuccess +backends: [Papiv2-Reference-Disk-Localization] + +files { + workflow: reference_disk/reference_disk_mounted_only_if_requested.wdl + options: reference_disk/reference_disk_test_true.options.json +} + +metadata { + workflowName: ReferenceDiskMountedOnlyIfRequested + status: Succeeded + "outputs.ReferenceDiskMountedOnlyIfRequested.disk_mounted": true +} diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_unspecified.test b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_unspecified.test new file mode 100644 index 00000000000..860159b594e --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/reference_disk_mounted_only_if_requested_unspecified.test @@ -0,0 +1,14 @@ +name: reference_disk_mounted_only_if_requested_unspecified +testFormat: workflowsuccess +backends: [Papiv2-Reference-Disk-Localization] + +files { + workflow: reference_disk/reference_disk_mounted_only_if_requested.wdl + options: reference_disk/reference_disk_test_unspecified.options.json +} + +metadata { + workflowName: ReferenceDiskMountedOnlyIfRequested + status: Succeeded + "outputs.ReferenceDiskMountedOnlyIfRequested.disk_mounted": false +} diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_true_options.test b/centaur/src/main/resources/standardTestCases/reference_disk_true_options.test index dab9c516164..6d957bf55b7 100644 --- a/centaur/src/main/resources/standardTestCases/reference_disk_true_options.test +++ b/centaur/src/main/resources/standardTestCases/reference_disk_true_options.test @@ -11,5 +11,7 @@ files { metadata { workflowName: wf_reference_disk_test status: Succeeded - "outputs.wf_reference_disk_test.is_input_file_a_symlink": true + "outputs.wf_reference_disk_test.is_broad_input_file_a_valid_symlink": true + "outputs.wf_reference_disk_test.is_nirvana_input_file_a_valid_symlink": true + "outputs.wf_reference_disk_test.is_nirvana_metachar_input_file_a_valid_symlink": true } diff --git a/centaur/src/main/resources/standardTestCases/reference_disk_unspecified_options.test b/centaur/src/main/resources/standardTestCases/reference_disk_unspecified_options.test index 441db8748f2..7625bcd7806 100644 --- a/centaur/src/main/resources/standardTestCases/reference_disk_unspecified_options.test +++ b/centaur/src/main/resources/standardTestCases/reference_disk_unspecified_options.test @@ -11,5 +11,7 @@ files { metadata { workflowName: wf_reference_disk_test status: Succeeded - "outputs.wf_reference_disk_test.is_input_file_a_symlink": false + "outputs.wf_reference_disk_test.is_broad_input_file_a_valid_symlink": false + "outputs.wf_reference_disk_test.is_nirvana_input_file_a_valid_symlink": false + "outputs.wf_reference_disk_test.is_nirvana_metachar_input_file_a_valid_symlink": false } diff --git a/centaur/src/main/resources/standardTestCases/relative_output_paths_colliding.test b/centaur/src/main/resources/standardTestCases/relative_output_paths_colliding.test index 82b01c6399d..2a6fca6793e 100644 --- a/centaur/src/main/resources/standardTestCases/relative_output_paths_colliding.test +++ b/centaur/src/main/resources/standardTestCases/relative_output_paths_colliding.test @@ -1,5 +1,6 @@ name: relative_output_paths_colliding testFormat: workflowfailure +tags: [batchexclude] files { workflow: relative_output_paths_colliding/workflow_output_paths_colliding.wdl diff --git a/centaur/src/main/resources/standardTestCases/retry_with_more_memory/retry_with_more_memory_after_137.wdl b/centaur/src/main/resources/standardTestCases/retry_with_more_memory/retry_with_more_memory_after_137.wdl new file mode 100644 index 00000000000..2fe434475c6 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/retry_with_more_memory/retry_with_more_memory_after_137.wdl @@ -0,0 +1,22 @@ +version 1.0 + +task imitate_oom_error { + command { + printf "Exception in thread "main" java.lang.OutOfMemoryError: testing\n\tat Test.main(Test.java:1)\n" >&2 + touch foo + exit 137 + } + output { + File foo = "foo" + } + runtime { + docker: "python:latest" + memory: "1 GB" + maxRetries: 2 + backend: "Papiv2" + } +} + +workflow retry_with_more_memory_after_137 { + call imitate_oom_error +} diff --git a/centaur/src/main/resources/standardTestCases/retry_with_more_memory_after_137.test b/centaur/src/main/resources/standardTestCases/retry_with_more_memory_after_137.test new file mode 100644 index 00000000000..a69290ca511 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/retry_with_more_memory_after_137.test @@ -0,0 +1,21 @@ +name: retry_with_more_memory_after_137 +testFormat: workflowfailure +backends: [Papiv2] + +files { + workflow: retry_with_more_memory/retry_with_more_memory_after_137.wdl + options: retry_with_more_memory/retry_with_more_memory.options +} + +metadata { + workflowName: retry_with_more_memory_after_137 + status: Failed + "failures.0.message": "Workflow failed" + "failures.0.causedBy.0.message": "stderr for job `retry_with_more_memory_after_137.imitate_oom_error:NA:3` contained one of the `memory-retry-error-keys: [OutOfMemory,Killed]` specified in the Cromwell config. Job might have run out of memory." + "retry_with_more_memory_after_137.imitate_oom_error.-1.1.executionStatus": "RetryableFailure" + "retry_with_more_memory_after_137.imitate_oom_error.-1.1.runtimeAttributes.memory": "1 GB" + "retry_with_more_memory_after_137.imitate_oom_error.-1.2.executionStatus": "RetryableFailure" + "retry_with_more_memory_after_137.imitate_oom_error.-1.2.runtimeAttributes.memory": "1.1 GB" + "retry_with_more_memory_after_137.imitate_oom_error.-1.3.executionStatus": "Failed" + "retry_with_more_memory_after_137.imitate_oom_error.-1.3.runtimeAttributes.memory": "1.2100000000000002 GB" +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes.test b/centaur/src/main/resources/standardTestCases/return_codes.test new file mode 100644 index 00000000000..7528064f4c8 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes.test @@ -0,0 +1,11 @@ +name: return_codes +testFormat: workflowsuccess + +files { + workflow: return_codes/return_codes.wdl +} + +metadata { + workflowName: ReturnCodeValidation + status: Succeeded +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl b/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl new file mode 100644 index 00000000000..21e1da37650 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes/return_codes.wdl @@ -0,0 +1,69 @@ +version development-1.1 + +workflow ReturnCodeValidation { + call ReturnCodeSet1 + call ReturnCodeSet2 + call ReturnCodeSet3 + call ReturnCodeString +} + +task ReturnCodeSet1 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + } +} + +task ReturnCodeSet2 { + meta { + volatile: true + } + + command <<< + exit 200 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1, 123, 200] + } +} + +task ReturnCodeSet3 { + meta { + volatile: true + } + + command <<< + exit 10 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: 10 + } +} + + +task ReturnCodeString { + meta { + volatile: true + } + + command <<< + exit 500 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: "*" + } +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test new file mode 100644 index 00000000000..20c07a6e051 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version.test @@ -0,0 +1,11 @@ +name: return_codes_invalid_on_old_wdl_version +testFormat: workflowfailure + +files { + workflow: return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl +} + +metadata { + workflowName: ReturnCodesInvalidOnOldWdl + status: Failed +} diff --git a/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl new file mode 100644 index 00000000000..296dc833f02 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/return_codes_invalid_on_old_wdl_version/return_codes_invalid_on_old_wdl_version.wdl @@ -0,0 +1,18 @@ +workflow ReturnCodesInvalidOnOldWdl { + call ReturnCodesInvalidOnOldWdlTask +} + +task ReturnCodesInvalidOnOldWdlTask { + meta { + volatile: "true" + } + + command <<< + exit 5 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [5, 10, 15] + } +} diff --git a/centaur/src/main/resources/standardTestCases/scatter-wf1.test b/centaur/src/main/resources/standardTestCases/scatter-wf1.test deleted file mode 100644 index 397b94710a3..00000000000 --- a/centaur/src/main/resources/standardTestCases/scatter-wf1.test +++ /dev/null @@ -1,11 +0,0 @@ -name: cwl_scatter_wf1 -testFormat: workflowsuccess -skipDescribeEndpointValidation: true - -files { - workflow: cwls/scatter-wf1.cwl - inputs: cwls/scatter-job1.json -} - -workflowType: CWL -workflowTypeVersion: v1.0 diff --git a/centaur/src/main/resources/standardTestCases/scatterchain.test b/centaur/src/main/resources/standardTestCases/scatterchain.test index 26fdb95b27b..c1b02be7abe 100644 --- a/centaur/src/main/resources/standardTestCases/scatterchain.test +++ b/centaur/src/main/resources/standardTestCases/scatterchain.test @@ -1,6 +1,5 @@ name: scatterchain testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: scatter_chain/scatter_chain.wdl diff --git a/centaur/src/main/resources/standardTestCases/scattergather.test b/centaur/src/main/resources/standardTestCases/scattergather.test index fad78c0a08c..425be6fd3c4 100644 --- a/centaur/src/main/resources/standardTestCases/scattergather.test +++ b/centaur/src/main/resources/standardTestCases/scattergather.test @@ -1,6 +1,5 @@ name: scattergather testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: scattergather/scattergather.wdl diff --git a/centaur/src/main/resources/standardTestCases/short_circuit.test b/centaur/src/main/resources/standardTestCases/short_circuit.test index a0727dc949f..a961f39071b 100644 --- a/centaur/src/main/resources/standardTestCases/short_circuit.test +++ b/centaur/src/main/resources/standardTestCases/short_circuit.test @@ -2,7 +2,7 @@ name: short_circuit testFormat: workflowsuccess backends: [Local] -tags: [localdockertest, "wdl_upgrade"] +tags: [localdockertest] files { diff --git a/centaur/src/main/resources/standardTestCases/square.test b/centaur/src/main/resources/standardTestCases/square.test index ef0ac9a9ef8..db4e5851ead 100644 --- a/centaur/src/main/resources/standardTestCases/square.test +++ b/centaur/src/main/resources/standardTestCases/square.test @@ -1,6 +1,5 @@ name: square testFormat: workflowsuccess -tags: [ "wdl_upgrade" ] files { workflow: square/square.wdl diff --git a/centaur/src/main/resources/standardTestCases/standard_output_paths_colliding_prevented.test b/centaur/src/main/resources/standardTestCases/standard_output_paths_colliding_prevented.test index 6c5a5b51476..d8d37b4b2d0 100644 --- a/centaur/src/main/resources/standardTestCases/standard_output_paths_colliding_prevented.test +++ b/centaur/src/main/resources/standardTestCases/standard_output_paths_colliding_prevented.test @@ -1,5 +1,6 @@ name: standard_output_paths_colliding_prevented testFormat: workflowsuccess +tags: [batchexclude] files { workflow: standard_output_paths_colliding_prevented/workflow_output_paths_colliding.wdl diff --git a/centaur/src/main/resources/standardTestCases/stderr_stdout_workflow_body.test b/centaur/src/main/resources/standardTestCases/stderr_stdout_workflow_body.test new file mode 100644 index 00000000000..71cd116b44a --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/stderr_stdout_workflow_body.test @@ -0,0 +1,13 @@ +name: stderr_stdout_workflow_body +testFormat: workflowfailure + +files { + workflow: failures/stderr_stdout_workflow_body/stderr_stdout_workflow_body.wdl +} + +metadata { + workflowName: break_with_stderr + status: Failed + "failures.0.message": "Workflow failed" + "failures.0.causedBy.0.message": "Failed to evaluate 'break_with_stderr.load_data_csv' (reason 1 of 2): Evaluating select_first([stdout(), stderr()]) failed: stdout is not implemented at the workflow level, Failed to evaluate 'break_with_stderr.load_data_csv' (reason 2 of 2): Evaluating select_first([stdout(), stderr()]) failed: stderr is not implemented at the workflow level" +} diff --git a/centaur/src/main/resources/standardTestCases/string_interpolation.test b/centaur/src/main/resources/standardTestCases/string_interpolation.test index 7e7e894f9fa..04736e59ea4 100644 --- a/centaur/src/main/resources/standardTestCases/string_interpolation.test +++ b/centaur/src/main/resources/standardTestCases/string_interpolation.test @@ -1,6 +1,5 @@ name: string_interpolation testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: string_interpolation/string_interpolation.wdl diff --git a/centaur/src/main/resources/standardTestCases/struct_literal.test b/centaur/src/main/resources/standardTestCases/struct_literal.test new file mode 100644 index 00000000000..72755145a02 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/struct_literal.test @@ -0,0 +1,12 @@ +name: struct_literal +testFormat: workflowsuccess + +files { + workflow: struct_literal/struct_literal.wdl +} + +metadata { + workflowName: struct_literal + status: Succeeded + "outputs.struct_literal.out": 44 +} diff --git a/centaur/src/main/resources/standardTestCases/struct_literal/struct_literal.wdl b/centaur/src/main/resources/standardTestCases/struct_literal/struct_literal.wdl new file mode 100644 index 00000000000..215fd40f8d3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/struct_literal/struct_literal.wdl @@ -0,0 +1,68 @@ +version development-1.1 + +struct Plant { + String color + Int id +} + +struct Fungi { + File fungiFile +} + + +struct Animal { + Plant jacket + Fungi hat +} + +task a { + input { + Plant in_plant_literal = Plant{color: "red", id: 44} + } + + command { + echo "${in_plant_literal.id}" + } + + output { + Animal out_animal = Animal{jacket: Plant{color: "green", id: 10}, hat: Fungi{fungiFile: stdout()}} + } + + runtime { + docker: "ubuntu:latest" + } + + meta { + volatile: true + } +} + +task b { + input { + Animal in_animal + } + + command { + cat ${in_animal.hat.fungiFile} + } + + output { + Int out = read_int(stdout()) + } + + runtime { + docker: "ubuntu:latest" + } + + meta { + volatile: true + } +} + +workflow struct_literal { + call a + call b {input: in_animal=a.out_animal} + output { + Int out = b.out + } +} diff --git a/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test b/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test index 4b216940cb2..cc18f753e42 100644 --- a/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test +++ b/centaur/src/main/resources/standardTestCases/sub_workflow_hello_world.test @@ -1,6 +1,6 @@ name: sub_workflow_hello_world testFormat: workflowsuccess -tags: [subworkflow, "wdl_upgrade"] +tags: [subworkflow] files { workflow: sub_workflow_hello_world/sub_workflow_hello_world.wdl diff --git a/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test b/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test index 390c089359b..deca93aa195 100644 --- a/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test +++ b/centaur/src/main/resources/standardTestCases/sub_workflow_var_refs.test @@ -1,6 +1,6 @@ name: sub_workflow_var_refs testFormat: workflowsuccess -tags: [subworkflow, "wdl_upgrade"] +tags: [subworkflow] files { workflow: sub_workflow_var_refs/sub_workflow_var_refs.wdl diff --git a/centaur/src/main/resources/standardTestCases/three_step.test b/centaur/src/main/resources/standardTestCases/three_step.test deleted file mode 100644 index b24e649ecce..00000000000 --- a/centaur/src/main/resources/standardTestCases/three_step.test +++ /dev/null @@ -1,21 +0,0 @@ -name: three_step_cwl -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 -skipDescribeEndpointValidation: true - -files { - workflow: three_step/three_step.cwl - inputs: three_step/inputs.json -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 - "calls.wc.executionStatus": Done - "calls.ps.executionStatus": Done - "calls.cgrep.executionStatus": Done - "calls.ps.runtimeAttributes.docker": "ubuntu:bionic" - "calls.wc.runtimeAttributes.docker": "ubuntu:latest" - "calls.cgrep.runtimeAttributes.docker": "debian:jessie" -} diff --git a/centaur/src/main/resources/standardTestCases/three_step/inputs.json b/centaur/src/main/resources/standardTestCases/three_step/inputs.json deleted file mode 100644 index 8dac4768fff..00000000000 --- a/centaur/src/main/resources/standardTestCases/three_step/inputs.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "pattern": "java" -} diff --git a/centaur/src/main/resources/standardTestCases/three_step/three_step.cwl b/centaur/src/main/resources/standardTestCases/three_step/three_step.cwl deleted file mode 100644 index e328003eab1..00000000000 --- a/centaur/src/main/resources/standardTestCases/three_step/three_step.cwl +++ /dev/null @@ -1,114 +0,0 @@ -cwlVersion: v1.0 -class: Workflow -id: three_step -# Workflow-level DockerRequirement -hints: - DockerRequirement: - dockerPull: "ubuntu:latest" -inputs: -- id: pattern - type: string -outputs: -- id: cgrep-count - outputSource: "#cgrep/cgrep-count" - type: int -- id: wc-count - outputSource: "#wc/wc-count" - type: int -steps: -- id: ps - in: [] - out: - - id: ps-stdOut - run: - inputs: [] - outputs: - - id: ps-stdOut - outputBinding: - glob: ps-stdOut.txt - type: File - class: CommandLineTool - requirements: - # Command line tool-level DockerRequirement - # Check it: this DockerRequirement does not use the `class` formatting but should still parse - # thanks to the magic of SALAD. - DockerRequirement: - dockerPull: "ubuntu:bionic" - baseCommand: ps - stdout: ps-stdOut.txt -- id: cgrep - in: - - id: pattern - source: "#pattern" - - id: file - source: "#ps/ps-stdOut" - out: - - id: cgrep-count - # Workflow step-level DockerRequirement - requirements: - DockerRequirement: - dockerPull: "debian:jessie" - run: - inputs: - - id: pattern - type: string - - id: file - type: File - outputs: - - id: cgrep-count - outputBinding: - glob: cgrep-stdOut.txt - loadContents: true - outputEval: $(parseInt(self[0].contents)) - type: int - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - - class: InlineJavascriptRequirement - arguments: - - valueFrom: grep - shellQuote: false - - valueFrom: $(inputs.pattern) - shellQuote: false - - valueFrom: ${return inputs.file} - shellQuote: false - - valueFrom: '|' - shellQuote: false - - valueFrom: wc - shellQuote: false - - valueFrom: -l - shellQuote: false - stdout: cgrep-stdOut.txt -- id: wc - in: - - id: file - source: "#ps/ps-stdOut" - out: - - id: wc-count - run: - inputs: - - id: file - type: File - outputs: - - id: wc-count - outputBinding: - glob: wc-stdOut.txt - loadContents: true - outputEval: $(parseInt(self[0].contents)) - type: int - class: CommandLineTool - requirements: - - class: ShellCommandRequirement - - class: InlineJavascriptRequirement - arguments: - - valueFrom: cat - shellQuote: false - - valueFrom: $(inputs.file) - shellQuote: false - - valueFrom: '|' - shellQuote: false - - valueFrom: wc - shellQuote: false - - valueFrom: -l - shellQuote: false - stdout: wc-stdOut.txt diff --git a/centaur/src/main/resources/standardTestCases/three_step/three_step_caller_wf.cwl b/centaur/src/main/resources/standardTestCases/three_step/three_step_caller_wf.cwl deleted file mode 100644 index ab5d8ddc458..00000000000 --- a/centaur/src/main/resources/standardTestCases/three_step/three_step_caller_wf.cwl +++ /dev/null @@ -1,24 +0,0 @@ -#!/usr/bin/env cwl-runner -class: Workflow -cwlVersion: v1.0 - -inputs: -- id: pin - type: string - default: "v" - -outputs: - count_output: - type: int - outputSource: threestep/wc-count - -requirements: - - class: SubworkflowFeatureRequirement - -steps: - threestep: - run: three_step.cwl - in: - - id: pattern - source: pin - out: [wc-count] diff --git a/centaur/src/main/resources/standardTestCases/three_step_subworkflow.test b/centaur/src/main/resources/standardTestCases/three_step_subworkflow.test deleted file mode 100644 index 6ae478035b7..00000000000 --- a/centaur/src/main/resources/standardTestCases/three_step_subworkflow.test +++ /dev/null @@ -1,16 +0,0 @@ -name: three_step__subwf_cwl -testFormat: workflowsuccess -workflowType: CWL -workflowTypeVersion: v1.0 - -files { - workflow: three_step/three_step_caller_wf.cwl, - imports: [ - three_step/three_step.cwl - ] -} - -metadata { - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 -} diff --git a/centaur/src/main/resources/standardTestCases/valid_labels.test b/centaur/src/main/resources/standardTestCases/valid_labels.test index 1ab748f8a86..6f7f4e7ceec 100644 --- a/centaur/src/main/resources/standardTestCases/valid_labels.test +++ b/centaur/src/main/resources/standardTestCases/valid_labels.test @@ -1,6 +1,6 @@ name: valid_labels testFormat: workflowsuccess -tags: [ labels, "wdl_upgrade" ] +tags: [ labels ] files { workflow: hello/hello.wdl diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl new file mode 100644 index 00000000000..41acb028221 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl @@ -0,0 +1,55 @@ +version development-1.1 + +workflow ValidReturnCodeAndContinueOnReturnCode { + call ReturnCodeContinueOnReturnCode1 + call ReturnCodeContinueOnReturnCode2 + call ReturnCodeContinueOnReturnCode3 +} + +task ReturnCodeContinueOnReturnCode1 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + continueOnReturnCode: [0] + } +} + +task ReturnCodeContinueOnReturnCode2 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1] + continueOnReturnCode: false + } +} + +task ReturnCodeContinueOnReturnCode3 { + meta { + volatile: true + } + + command <<< + exit 1 + >>> + + runtime { + docker: "ubuntu:latest" + returnCodes: [1, 4, 7] + continueOnReturnCode: [1, 3, 5] + } +} diff --git a/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test new file mode 100644 index 00000000000..1a8354b9482 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/valid_return_codes_and_continue_on_return_codes.test @@ -0,0 +1,11 @@ +name: valid_return_codes_and_continue_on_return_code +testFormat: workflowsuccess + +files { + workflow: valid_return_codes_and_continue_on_return_code/valid_return_codes_and_continue_on_return_code.wdl +} + +metadata { + workflowName: ValidReturnCodeAndContinueOnReturnCode + status: Succeeded +} diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_http_relative_imports/biscayne_http_relative_imports.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_http_relative_imports/biscayne_http_relative_imports.wdl index c75ffacc8a8..dbe4fdee0f8 100644 --- a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_http_relative_imports/biscayne_http_relative_imports.wdl +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_http_relative_imports/biscayne_http_relative_imports.wdl @@ -1,4 +1,4 @@ -version development +version development-1.1 # Is there a better way to test this? import "https://raw.githubusercontent.com/broadinstitute/cromwell/develop/womtool/src/test/resources/validate/biscayne/valid/relative_imports/sub_wfs/foo.wdl" diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl index 979ef6e998a..979af2dc778 100644 --- a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_engine_functions/biscayne_new_engine_functions.wdl @@ -1,10 +1,10 @@ -version development +version development-1.1 workflow biscayne_new_engine_functions { meta { description: "This test makes sure that these functions work in a real workflow" - functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key" ] + functions_under_test: [ "keys", "as_map", "as_pairs", "collect_by_key", "quote", "squote", "sub", "suffix", "unzip" ] } Map[String, Int] x_map_in = {"a": 1, "b": 2, "c": 3} @@ -15,6 +15,10 @@ workflow biscayne_new_engine_functions { Array[Pair[String,Int]] z_pairs_in = [("a", 1), ("b", 2), ("a", 3)] + Array[String] some_strings = ["aaa", "bbb", "ccc"] + + Array[Int] some_ints = [1, 2, 3] + Int smallestInt = 1 Float smallFloat = 2.718 Float bigFloat = 3.141 @@ -24,6 +28,10 @@ workflow biscayne_new_engine_functions { # max float... near enough: Float maxFloat = 179769313000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000.0 + Array[Pair[String,String]] zipped_a = [("A", "a")] + Array[Pair[String,String]] zipped_b = [("A", "a"),("B", "b")] + Array[Pair[String,Float]] zipped_c = [("one", 1.0),("two", 2.0),("three", 3.0)] + output { # keys(), as_map(), as_pairs(), collect_by_key(): @@ -48,6 +56,31 @@ workflow biscayne_new_engine_functions { Float bigIntFloatComparison = max(bigFloat, biggestInt) # 10.0 Float minMaxIntFloatComposition = min(max(biggestInt, smallFloat), smallestInt) # 1.0 Float maxIntVsMaxFloat = max(maxInt, maxFloat) + + # sub(): + # (Exists before Biscayne, but uses different regex flavor here) + # ================================================= + String substituted = sub("AtheZ", "[[:upper:]]", "WAT") + + # suffix(): + # ================================================= + Array[String] with_suffixes = suffix("S", some_strings) + + # quote(): + # ================================================= + Array[String] with_quotes = quote(some_ints) + Array[String] string_with_quotes = quote(some_strings) + + # squote(): + # ================================================= + Array[String] with_squotes = squote(some_ints) + Array[String] string_with_squotes = squote(some_strings) + + # unzip(): + # ================================================= + Pair[Array[String], Array[String]] unzipped_a = unzip(zipped_a) + Pair[Array[String], Array[String]] unzipped_b = unzip(zipped_b) + Pair[Array[String], Array[String]] unzipped_c = unzip(zipped_c) } } diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl new file mode 100644 index 00000000000..1174d5c02d6 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_new_runtime_attributes/biscayne_new_runtime_attributes.wdl @@ -0,0 +1,48 @@ +version development-1.1 + +workflow runtime_attributes_wf { + call runtime_attributes_task + output { + String out = runtime_attributes_task.out + } +} + +task runtime_attributes_task { + + command <<< + echo "Zardoz" + >>> + + meta { + volatile: true + } + + runtime { + # Meaningless keys are ignored + banana: object { + cpuPlatform: "Banana Lake" + } + + gcp: object { + # Platform-specific keys take precedence + docker: "rockylinux:9", + memory: "6 GB" + } + + azure: object { + memory: "4 GB", + docker: "debian:latest" + } + + # Generic keys are ignored in favor of platform ones + docker: "ubuntu:latest" + memory: "8 GB" + + # We still read generic keys that are not overridden + cpu: 4 + } + + output { + String out = read_string(stdout()) + } +} diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_prohibits_directory/directory_type.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_prohibits_directory/directory_type.wdl new file mode 100644 index 00000000000..7bd7f5583c3 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_prohibits_directory/directory_type.wdl @@ -0,0 +1,49 @@ +version development-1.1 + +workflow directory_type { + input { + String text2loc = "text2" + } + call make_directory { input: text2loc = text2loc } + call read_from_directory { input: d = make_directory.d, text2loc = text2loc } + + output { + Array[String] out = read_from_directory.contents + } +} + +task make_directory { + input { + String text2loc + } + String text2dir = sub("foo/~{text2loc}", "/[^/]*$", "") + command { + mkdir foo + mkdir -p ~{text2dir} + echo "foo text" > foo/text + echo "foo text2" > foo/~{text2loc} + } + runtime { + docker: "ubuntu:latest" + } + output { + Directory d = "foo/" + } +} + +task read_from_directory { + input { + String text2loc + Directory d + } + command { + cat ~{d}/text + cat ~{d}/~{text2loc} + } + runtime { + docker: "ubuntu:latest" + } + output { + Array[String] contents = read_lines(stdout()) + } +} diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/directory_type/directory_type_local_inputs.json b/centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_prohibits_directory/directory_type_local_inputs.json similarity index 100% rename from centaur/src/main/resources/standardTestCases/wdl_biscayne/directory_type/directory_type_local_inputs.json rename to centaur/src/main/resources/standardTestCases/wdl_biscayne/biscayne_prohibits_directory/directory_type_local_inputs.json diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/default_default/default_default.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/default_default/default_default.wdl index 20958128db5..a73b3736fdf 100644 --- a/centaur/src/main/resources/standardTestCases/wdl_biscayne/default_default/default_default.wdl +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/default_default/default_default.wdl @@ -1,4 +1,4 @@ -version development +version development-1.1 workflow default_default { call default_default_task diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/read_functions_windows_line_endings/read_functions_windows_line_endings.wdl b/centaur/src/main/resources/standardTestCases/wdl_biscayne/read_functions_windows_line_endings/read_functions_windows_line_endings.wdl index 033bdfc4d97..64b792fd5dd 100644 --- a/centaur/src/main/resources/standardTestCases/wdl_biscayne/read_functions_windows_line_endings/read_functions_windows_line_endings.wdl +++ b/centaur/src/main/resources/standardTestCases/wdl_biscayne/read_functions_windows_line_endings/read_functions_windows_line_endings.wdl @@ -1,4 +1,4 @@ -version development +version development-1.1 struct JsonObj { String field1 diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/directory_type/directory_type.wdl b/centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type/directory_type.wdl similarity index 100% rename from centaur/src/main/resources/standardTestCases/wdl_biscayne/directory_type/directory_type.wdl rename to centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type/directory_type.wdl diff --git a/centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type/directory_type_local_inputs.json b/centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type/directory_type_local_inputs.json new file mode 100644 index 00000000000..cfe1a8bb688 --- /dev/null +++ b/centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type/directory_type_local_inputs.json @@ -0,0 +1,3 @@ +{ + "directory_type.text2loc": "bar/text2" +} diff --git a/centaur/src/main/resources/standardTestCases/wdl_biscayne/directory_type_output/directory_type_output.wdl b/centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type_output/directory_type_output.wdl similarity index 100% rename from centaur/src/main/resources/standardTestCases/wdl_biscayne/directory_type_output/directory_type_output.wdl rename to centaur/src/main/resources/standardTestCases/wdl_cascades/directory_type_output/directory_type_output.wdl diff --git a/centaur/src/main/resources/standardTestCases/workflow_type_and_version_cwl.test b/centaur/src/main/resources/standardTestCases/workflow_type_and_version_cwl.test deleted file mode 100644 index 14ba92c588f..00000000000 --- a/centaur/src/main/resources/standardTestCases/workflow_type_and_version_cwl.test +++ /dev/null @@ -1,16 +0,0 @@ -ignore: true -name: workflow_type_and_version_cwl -testFormat: workflowsuccess - -files { - workflow: workflow_type_and_version_cwl/workflow_type_and_version_cwl.wdl -} - -workflowType: CWL -workflowTypeVersion: v1.0 - -metadata { - workflowName: workflow_type_and_version_cwl - "submittedFiles.workflowType": CWL - "submittedFiles.workflowTypeVersion": v1.0 -} diff --git a/centaur/src/main/resources/standardTestCases/write_lines.test b/centaur/src/main/resources/standardTestCases/write_lines.test index 977459a50c7..fa0c6edd973 100644 --- a/centaur/src/main/resources/standardTestCases/write_lines.test +++ b/centaur/src/main/resources/standardTestCases/write_lines.test @@ -1,6 +1,5 @@ name: write_lines testFormat: workflowsuccess -tags: ["wdl_upgrade"] files { workflow: write_lines/write_lines.wdl @@ -13,4 +12,3 @@ metadata { "calls.write_lines.f2a.executionStatus": Done "outputs.write_lines.a2f_second.x": "a\nb\nc\nd" } - diff --git a/centaur/src/main/scala/centaur/CentaurConfig.scala b/centaur/src/main/scala/centaur/CentaurConfig.scala index f17c288283a..c15ae31f453 100644 --- a/centaur/src/main/scala/centaur/CentaurConfig.scala +++ b/centaur/src/main/scala/centaur/CentaurConfig.scala @@ -41,16 +41,19 @@ sealed trait CentaurRunMode { def cromwellUrl: URL } -case class UnmanagedCromwellServer(cromwellUrl : URL) extends CentaurRunMode -case class ManagedCromwellServer(preRestart: CromwellConfiguration, postRestart: CromwellConfiguration, withRestart: Boolean) extends CentaurRunMode { +case class UnmanagedCromwellServer(cromwellUrl: URL) extends CentaurRunMode +case class ManagedCromwellServer(preRestart: CromwellConfiguration, + postRestart: CromwellConfiguration, + withRestart: Boolean +) extends CentaurRunMode { override val cromwellUrl = new URL(s"http://localhost:${CromwellManager.ManagedCromwellPort}") } object CentaurConfig { lazy val conf: Config = ConfigFactory.load().getConfig("centaur") - + lazy val runMode: CentaurRunMode = CentaurRunMode(conf) - + lazy val cromwellUrl: URL = runMode.cromwellUrl lazy val workflowProgressTimeout: FiniteDuration = conf.getDuration("workflow-progress-timeout").toScala lazy val sendReceiveTimeout: FiniteDuration = conf.getDuration("sendReceiveTimeout").toScala diff --git a/centaur/src/main/scala/centaur/CromwellManager.scala b/centaur/src/main/scala/centaur/CromwellManager.scala index a53416a0ded..55c317b41b3 100644 --- a/centaur/src/main/scala/centaur/CromwellManager.scala +++ b/centaur/src/main/scala/centaur/CromwellManager.scala @@ -18,15 +18,15 @@ object CromwellManager extends StrictLogging { private var cromwellProcess: Option[CromwellProcess] = None private var _ready: Boolean = false private var _isManaged: Boolean = false - + /** * Returns true if Cromwell is ready to be queried, false otherwise * In Unmanaged mode, this is irrelevant so always return true. * In managed mode return the value of _ready */ def isReady: Boolean = !_isManaged || _ready - - // Check that we have a cromwellProcess, that this process is alive, and that cromwell is ready to accept requests + + // Check that we have a cromwellProcess, that this process is alive, and that cromwell is ready to accept requests private def isAlive(checkType: String): Boolean = { val processAlive = cromwellProcess.exists(_.isAlive) logger.info(s"Cromwell process alive $checkType = $processAlive") @@ -76,14 +76,14 @@ object CromwellManager extends StrictLogging { def stopCromwell(reason: String) = { _ready = false logger.info(s"Stopping Cromwell... ($reason)") - try { + try cromwellProcess foreach { _.stop() } - } catch { - case e: Exception => + catch { + case e: Exception => logger.error("Caught exception while stopping Cromwell") e.printStackTrace() } - + cromwellProcess = None } } diff --git a/centaur/src/main/scala/centaur/CromwellTracker.scala b/centaur/src/main/scala/centaur/CromwellTracker.scala index 4a89dea302d..1ec6fefe54c 100644 --- a/centaur/src/main/scala/centaur/CromwellTracker.scala +++ b/centaur/src/main/scala/centaur/CromwellTracker.scala @@ -14,7 +14,6 @@ import org.apache.commons.math3.stat.inference.ChiSquareTest import scala.language.postfixOps - case class CromwellTracker(backendCount: Int, configuredSignificance: Double) extends StrictLogging { var counts: Map[String, Int] = Map() def track(metadata: WorkflowMetadata): Unit = { @@ -45,10 +44,15 @@ case class CromwellTracker(backendCount: Int, configuredSignificance: Double) ex val actual: Array[Long] = counts.values map { _.toLong } toArray val observedSignificance = new ChiSquareTest().chiSquareTest(expected, actual) - logger.info(f"configured/observed horicromtal significance levels: $configuredSignificance%.4f/$observedSignificance%.4f", configuredSignificance, observedSignificance) + logger.info( + f"configured/observed horicromtal significance levels: $configuredSignificance%.4f/$observedSignificance%.4f", + configuredSignificance, + observedSignificance + ) if (observedSignificance < configuredSignificance) { - val message = f"Failed horicromtal check: observed significance level $observedSignificance%.4f, minimum of $configuredSignificance%.4f was required" + val message = + f"Failed horicromtal check: observed significance level $observedSignificance%.4f, minimum of $configuredSignificance%.4f was required" throw new RuntimeException(message) } } diff --git a/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala b/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala index 6fab385537b..eeb1df1557f 100644 --- a/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala +++ b/centaur/src/main/scala/centaur/DockerComposeCromwellConfiguration.scala @@ -14,13 +14,17 @@ object DockerComposeCromwellConfiguration { } } -case class DockerComposeCromwellConfiguration(dockerTag: String, dockerComposeFile: String, conf: String, logFile: String) extends CromwellConfiguration { +case class DockerComposeCromwellConfiguration(dockerTag: String, + dockerComposeFile: String, + conf: String, + logFile: String +) extends CromwellConfiguration { override def createProcess: CromwellProcess = { - case class DockerComposeCromwellProcess(override val cromwellConfiguration: DockerComposeCromwellConfiguration) extends CromwellProcess { + case class DockerComposeCromwellProcess(override val cromwellConfiguration: DockerComposeCromwellConfiguration) + extends CromwellProcess { - private def composeCommand(command: String*): Array[String] = { + private def composeCommand(command: String*): Array[String] = Array("docker-compose", "-f", dockerComposeFile) ++ command - } private val startCommand = composeCommand("up", "--abort-on-container-exit") private val logsCommand = composeCommand("logs") @@ -29,14 +33,13 @@ case class DockerComposeCromwellConfiguration(dockerTag: String, dockerComposeFi private val envVariables = Map[String, String]( "CROMWELL_BUILD_CENTAUR_MANAGED_PORT" -> ManagedCromwellPort.toString, "CROMWELL_BUILD_CENTAUR_MANAGED_TAG" -> dockerTag, - "CROMWELL_BUILD_CENTAUR_MANAGED_CONFIG" -> conf, + "CROMWELL_BUILD_CENTAUR_MANAGED_CONFIG" -> conf ) private var process: Option[Process] = None - override def start(): Unit = { + override def start(): Unit = process = Option(runProcess(startCommand, envVariables)) - } override def stop(): Unit = { if (!isAlive) { diff --git a/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala b/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala index c95316d1a32..1bf83b16ee1 100644 --- a/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala +++ b/centaur/src/main/scala/centaur/JarCromwellConfiguration.scala @@ -15,20 +15,15 @@ object JarCromwellConfiguration { case class JarCromwellConfiguration(jar: String, conf: String, logFile: String) extends CromwellConfiguration { override def createProcess: CromwellProcess = { - case class JarCromwellProcess(override val cromwellConfiguration: JarCromwellConfiguration) extends CromwellProcess { - private val command = Array( - "java", - s"-Dconfig.file=$conf", - s"-Dwebservice.port=$ManagedCromwellPort", - "-jar", - jar, - "server") + case class JarCromwellProcess(override val cromwellConfiguration: JarCromwellConfiguration) + extends CromwellProcess { + private val command = + Array("java", s"-Dconfig.file=$conf", s"-Dwebservice.port=$ManagedCromwellPort", "-jar", jar, "server") private var process: Option[Process] = None - override def start(): Unit = { + override def start(): Unit = process = Option(runProcess(command, Map.empty)) - } override def stop(): Unit = { process foreach { @@ -37,7 +32,7 @@ case class JarCromwellConfiguration(jar: String, conf: String, logFile: String) process = None } - override def isAlive: Boolean = process.exists { _.isAlive } + override def isAlive: Boolean = process.exists(_.isAlive) override def logFile: String = cromwellConfiguration.logFile } diff --git a/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala b/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala index 33f96974b42..defc30bc108 100644 --- a/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala +++ b/centaur/src/main/scala/centaur/api/CentaurCromwellClient.scala @@ -30,20 +30,20 @@ object CentaurCromwellClient extends StrictLogging { // Do not use scala.concurrent.ExecutionContext.Implicits.global as long as this is using Await.result // See https://github.com/akka/akka-http/issues/602 // And https://github.com/viktorklang/blog/blob/master/Futures-in-Scala-2.12-part-7.md - final implicit val blockingEc: ExecutionContextExecutor = ExecutionContext.fromExecutor( - Executors.newFixedThreadPool(100, DaemonizedDefaultThreadFactory)) + implicit final val blockingEc: ExecutionContextExecutor = + ExecutionContext.fromExecutor(Executors.newFixedThreadPool(100, DaemonizedDefaultThreadFactory)) // Akka HTTP needs both the actor system and a materializer - final implicit val system: ActorSystem = ActorSystem("centaur-acting-like-a-system") - final implicit val materializer: ActorMaterializer = ActorMaterializer(ActorMaterializerSettings(system)) + implicit final val system: ActorSystem = ActorSystem("centaur-acting-like-a-system") + implicit final val materializer: ActorMaterializer = ActorMaterializer(ActorMaterializerSettings(system)) final val apiVersion = "v1" val cromwellClient = new CromwellClient(CentaurConfig.cromwellUrl, apiVersion) val defaultMetadataArgs: Option[Map[String, List[String]]] = config.getAs[Map[String, List[String]]]("centaur.metadata-args") - def submit(workflow: Workflow): IO[SubmittedWorkflow] = { - sendReceiveFutureCompletion(() => { + def submit(workflow: Workflow): IO[SubmittedWorkflow] = + sendReceiveFutureCompletion { () => val submitted = cromwellClient.submit(workflow.toWorkflowSubmission) submitted.biSemiflatMap( httpResponse => @@ -59,44 +59,40 @@ object CentaurCromwellClient extends StrictLogging { _ = workflow.submittedWorkflowTracker.add(submittedWorkflow) } yield submittedWorkflow ) - }) - } + } - def describe(workflow: Workflow): IO[WaasDescription] = { + def describe(workflow: Workflow): IO[WaasDescription] = sendReceiveFutureCompletion(() => cromwellClient.describe(workflow.toWorkflowDescribeRequest)) - } - def status(workflow: SubmittedWorkflow): IO[WorkflowStatus] = { + def status(workflow: SubmittedWorkflow): IO[WorkflowStatus] = sendReceiveFutureCompletion(() => cromwellClient.status(workflow.id)) - } - def abort(workflow: SubmittedWorkflow): IO[WorkflowStatus] = { + def abort(workflow: SubmittedWorkflow): IO[WorkflowStatus] = sendReceiveFutureCompletion(() => cromwellClient.abort(workflow.id)) - } - def outputs(workflow: SubmittedWorkflow): IO[WorkflowOutputs] = { + def outputs(workflow: SubmittedWorkflow): IO[WorkflowOutputs] = sendReceiveFutureCompletion(() => cromwellClient.outputs(workflow.id)) - } - def callCacheDiff(workflowA: SubmittedWorkflow, callA: String, workflowB: SubmittedWorkflow, callB: String): IO[CallCacheDiff] = { - sendReceiveFutureCompletion(() => cromwellClient.callCacheDiff(workflowA.id, callA, ShardIndex(None), workflowB.id, callB, ShardIndex(None))) - } + def callCacheDiff(workflowA: SubmittedWorkflow, + callA: String, + workflowB: SubmittedWorkflow, + callB: String + ): IO[CallCacheDiff] = + sendReceiveFutureCompletion(() => + cromwellClient.callCacheDiff(workflowA.id, callA, ShardIndex(None), workflowB.id, callB, ShardIndex(None)) + ) - def logs(workflow: SubmittedWorkflow): IO[WorkflowMetadata] = { + def logs(workflow: SubmittedWorkflow): IO[WorkflowMetadata] = sendReceiveFutureCompletion(() => cromwellClient.logs(workflow.id)) - } - def labels(workflow: SubmittedWorkflow): IO[WorkflowLabels] = { + def labels(workflow: SubmittedWorkflow): IO[WorkflowLabels] = sendReceiveFutureCompletion(() => cromwellClient.labels(workflow.id)) - } - def addLabels(workflow: SubmittedWorkflow, newLabels: List[Label]): IO[WorkflowLabels] = { + def addLabels(workflow: SubmittedWorkflow, newLabels: List[Label]): IO[WorkflowLabels] = sendReceiveFutureCompletion(() => cromwellClient.addLabels(workflow.id, newLabels)) - } - def version: IO[CromwellVersion] = { + def version: IO[CromwellVersion] = sendReceiveFutureCompletion(() => cromwellClient.version) - } /* Sends a quick ping to the Cromwell query endpoint. The query endpoint is the only one which both hits the @@ -104,7 +100,9 @@ object CentaurCromwellClient extends StrictLogging { currently does not support query. */ def isAlive: Boolean = { - val response = Http().singleRequest(HttpRequest(uri=s"${CentaurConfig.cromwellUrl}/api/workflows/$apiVersion/query?status=Succeeded")) + val response = Http().singleRequest( + HttpRequest(uri = s"${CentaurConfig.cromwellUrl}/api/workflows/$apiVersion/query?status=Succeeded") + ) // Silence the following warning by discarding the result of a successful query: // Response entity was not subscribed after 1 second. Make sure to read the response entity body or call `discardBytes()` on it. val successOrFailure = response map { _.entity.discardBytes() } @@ -113,18 +111,19 @@ object CentaurCromwellClient extends StrictLogging { def metadata(workflow: SubmittedWorkflow, args: Option[Map[String, List[String]]] = defaultMetadataArgs, - expandSubworkflows: Boolean = false): IO[WorkflowMetadata] = { + expandSubworkflows: Boolean = false + ): IO[WorkflowMetadata] = { val mandatoryArgs = Map("expandSubWorkflows" -> List(expandSubworkflows.toString)) metadataWithId(workflow.id, Option(args.getOrElse(Map.empty) ++ mandatoryArgs)) } - def metadataWithId(id: WorkflowId, args: Option[Map[String, List[String]]] = defaultMetadataArgs): IO[WorkflowMetadata] = { + def metadataWithId(id: WorkflowId, + args: Option[Map[String, List[String]]] = defaultMetadataArgs + ): IO[WorkflowMetadata] = sendReceiveFutureCompletion(() => cromwellClient.metadata(id, args)) - } - def archiveStatus(id: WorkflowId): IO[String] = { + def archiveStatus(id: WorkflowId): IO[String] = sendReceiveFutureCompletion(() => cromwellClient.query(id)).map(_.results.head.metadataArchiveStatus) - } implicit private val timer: Timer[IO] = IO.timer(blockingEc) implicit private val contextShift: ContextShift[IO] = IO.contextShift(blockingEc) @@ -137,42 +136,43 @@ object CentaurCromwellClient extends StrictLogging { val stackTraceString = ExceptionUtils.getStackTrace(new Exception) - ioDelay.flatMap( _ => + ioDelay.flatMap(_ => // Could probably use IO to do the retrying too. For now use a copyport of Retry from cromwell core. Retry 5 times, // wait 5 seconds between retries. Timeout the whole thing using the IO timeout. // https://github.com/cb372/cats-retry // https://typelevel.org/cats-effect/datatypes/io.html#example-retrying-with-exponential-backoff - IO.fromFuture(IO(Retry.withRetry( - () => func().asIo.unsafeToFuture(), - Option(5), - 5.seconds, - isTransient = isTransient, - isFatal = isFatal - )).timeoutTo(timeout, - { + IO.fromFuture( + IO( + Retry.withRetry( + () => func().asIo.unsafeToFuture(), + Option(5), + 5.seconds, + isTransient = isTransient, + isFatal = isFatal + ) + ).timeoutTo( + timeout, IO.raiseError(new TimeoutException("Timeout from retryRequest " + timeout.toString + ": " + stackTraceString)) - } - ))) + ) + ) + ) } - def sendReceiveFutureCompletion[T](x: () => FailureResponseOrT[T]): IO[T] = { + def sendReceiveFutureCompletion[T](x: () => FailureResponseOrT[T]): IO[T] = retryRequest(x, CentaurConfig.sendReceiveTimeout) - } private def isFatal(f: Throwable) = f match { case _: DeserializationException => true case _ => false } - private def isTransient(f: Throwable) = { + private def isTransient(f: Throwable) = f match { - case _: StreamTcpException | - _: IOException | - _: UnsupportedContentTypeException => true + case _: StreamTcpException | _: IOException | _: UnsupportedContentTypeException => true case BufferOverflowException(message) => message.contains("Please retry the request later.") case unsuccessful: UnsuccessfulRequestException => unsuccessful.httpResponse.status == StatusCodes.NotFound - case unexpected: RuntimeException => unexpected.getMessage.contains("The http server closed the connection unexpectedly") + case unexpected: RuntimeException => + unexpected.getMessage.contains("The http server closed the connection unexpectedly") case _ => false } - } } diff --git a/centaur/src/main/scala/centaur/api/Retry.scala b/centaur/src/main/scala/centaur/api/Retry.scala index f384bc37684..1cbee9a1901 100644 --- a/centaur/src/main/scala/centaur/api/Retry.scala +++ b/centaur/src/main/scala/centaur/api/Retry.scala @@ -7,6 +7,7 @@ import scala.concurrent.duration._ import scala.concurrent.{ExecutionContext, Future} object Retry { + /** * Copied from cromwell.core * Replaced the backoff with a fixed retry delay @@ -16,8 +17,8 @@ object Retry { delay: FiniteDuration, isTransient: Throwable => Boolean = throwableToFalse, isFatal: Throwable => Boolean = throwableToFalse, - onRetry: Throwable => Unit = noopOnRetry) - (implicit actorSystem: ActorSystem): Future[A] = { + onRetry: Throwable => Unit = noopOnRetry + )(implicit actorSystem: ActorSystem): Future[A] = { // In the future we might want EC passed in separately but at the moment it caused more issues than it solved to do so implicit val ec: ExecutionContext = actorSystem.dispatcher @@ -25,10 +26,18 @@ object Retry { case throwable if isFatal(throwable) => Future.failed(throwable) case throwable if !isFatal(throwable) => val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 } - + if (retriesLeft.forall(_ > 0)) { onRetry(throwable) - after(delay, actorSystem.scheduler)(withRetry(f, delay = delay, maxRetries = retriesLeft, isTransient = isTransient, isFatal = isFatal, onRetry = onRetry)) + after(delay, actorSystem.scheduler)( + withRetry(f, + delay = delay, + maxRetries = retriesLeft, + isTransient = isTransient, + isFatal = isFatal, + onRetry = onRetry + ) + ) } else { Future.failed(throwable) } @@ -38,4 +47,3 @@ object Retry { def throwableToFalse(t: Throwable) = false def noopOnRetry(t: Throwable) = {} } - diff --git a/centaur/src/main/scala/centaur/json/JsonUtils.scala b/centaur/src/main/scala/centaur/json/JsonUtils.scala index 1af725d595b..90e3bd91a98 100644 --- a/centaur/src/main/scala/centaur/json/JsonUtils.scala +++ b/centaur/src/main/scala/centaur/json/JsonUtils.scala @@ -8,6 +8,7 @@ object JsonUtils { val attemptNumber = "attempt" implicit class EnhancedJsValue(val jsValue: JsValue) extends AnyVal { + /** * Modified from http://stackoverflow.com/a/31592156 - changes were made both to port from using * Play-JSON to Spray-JSON as well as to handle some cases specific to Cromwell's metadata response @@ -32,9 +33,8 @@ object JsonUtils { */ def flatten(prefix: String = ""): JsObject = { - def flattenShardAndAttempt(k:String, v: JsArray, f: JsObject => String): JsObject = { - v.elements.map(_.asJsObject).fold(JsObject.empty) { (x, y) => x ++ y.flatten(s"$k.${f(y)}") } - } + def flattenShardAndAttempt(k: String, v: JsArray, f: JsObject => String): JsObject = + v.elements.map(_.asJsObject).fold(JsObject.empty)((x, y) => x ++ y.flatten(s"$k.${f(y)}")) jsValue.asJsObject.fields.foldLeft(JsObject.empty) { case (acc, (k, v: JsArray)) if v.isSingleCallArray => acc ++ JsObject(k -> v.elements.head).flatten(prefix) @@ -44,10 +44,13 @@ object JsonUtils { to avoid lossy conversion for multiple attempts of the same shard. The older way of flattening shards with only shard index in the flattened structure is also kept so that the new structure doesn't fail tests that rely on the older flattened structure. This should be cleaned up in https://broadworkbench.atlassian.net/browse/BW-483 - */ + */ acc ++ flattenShardAndAttempt(k, v, (y: JsObject) => y.getField(shardIndex).get) ++ - flattenShardAndAttempt(k, v, (y: JsObject) => s"${y.getField(shardIndex).get}.${y.getField(attemptNumber).get}") + flattenShardAndAttempt(k, + v, + (y: JsObject) => s"${y.getField(shardIndex).get}.${y.getField(attemptNumber).get}" + ) case (acc, (k, v: JsArray)) => v.elements.zipWithIndex.foldLeft(acc) { case (accumulator, (element, idx)) => val maybePrefix = if (prefix.isEmpty) "" else s"$prefix." @@ -76,7 +79,7 @@ object JsonUtils { // A couple of helper functions to assist with flattening Cromwell metadata responses def hasField(fieldName: String): Boolean = jsObject.fields.keySet contains fieldName def getField(fieldName: String): Option[String] = jsObject.fields.get(fieldName) map { _.toString() } - def flattenToMap: Map [String, JsValue] = jsObject.flatten().fields map { case (k, v: JsValue) => k -> v} + def flattenToMap: Map[String, JsValue] = jsObject.flatten().fields map { case (k, v: JsValue) => k -> v } } /** @@ -89,9 +92,8 @@ object JsonUtils { def nonEmptyObjectArray = jsArray.isObjectArray && jsArray.nonEmpty def isSingleCallArray = jsArray.hasField(shardIndex) && jsArray.size == 1 - def hasField(fieldName: String): Boolean = { + def hasField(fieldName: String): Boolean = if (jsArray.nonEmptyObjectArray) jsArray.elements.map(_.asJsObject) forall { _.hasField(fieldName) } else false - } } } diff --git a/centaur/src/main/scala/centaur/test/CentaurTestException.scala b/centaur/src/main/scala/centaur/test/CentaurTestException.scala index fa83a3afb1a..ffc50a8dfdb 100644 --- a/centaur/src/main/scala/centaur/test/CentaurTestException.scala +++ b/centaur/src/main/scala/centaur/test/CentaurTestException.scala @@ -12,12 +12,12 @@ import cromwell.api.model.{SubmittedWorkflow, WorkflowMetadata} * @param metadataJsonOption The optional metadata. * @param causeOption The optional underlying cause. */ -case class CentaurTestException private(message: String, - testName: String, - workflowIdOption: Option[String], - metadataJsonOption: Option[String], - causeOption: Option[Exception]) - extends RuntimeException(message, causeOption.orNull) +case class CentaurTestException private (message: String, + testName: String, + workflowIdOption: Option[String], + metadataJsonOption: Option[String], + causeOption: Option[Exception] +) extends RuntimeException(message, causeOption.orNull) object CentaurTestException { @@ -25,7 +25,8 @@ object CentaurTestException { def apply(message: String, workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow, - actualMetadata: WorkflowMetadata): CentaurTestException = { + actualMetadata: WorkflowMetadata + ): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -33,12 +34,9 @@ object CentaurTestException { Option(actualMetadata.value), None ) - } /** Create a new CentaurTestException for a submitted workflow. */ - def apply(message: String, - workflowDefinition: Workflow, - submittedWorkflow: SubmittedWorkflow): CentaurTestException = { + def apply(message: String, workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -46,10 +44,9 @@ object CentaurTestException { None, None ) - } /** Create a new CentaurTestException for only a workflow definition. */ - def apply(message: String, workflowDefinition: Workflow): CentaurTestException = { + def apply(message: String, workflowDefinition: Workflow): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -57,10 +54,9 @@ object CentaurTestException { None, None ) - } /** Create a new CentaurTestException for only a workflow definition, including a root cause. */ - def apply(message: String, workflowDefinition: Workflow, cause: Exception): CentaurTestException = { + def apply(message: String, workflowDefinition: Workflow, cause: Exception): CentaurTestException = new CentaurTestException( message, workflowDefinition.testName, @@ -68,5 +64,4 @@ object CentaurTestException { None, Option(cause) ) - } } diff --git a/centaur/src/main/scala/centaur/test/FilesChecker.scala b/centaur/src/main/scala/centaur/test/FilesChecker.scala index 240552bb755..5f3f1d9e170 100644 --- a/centaur/src/main/scala/centaur/test/FilesChecker.scala +++ b/centaur/src/main/scala/centaur/test/FilesChecker.scala @@ -41,3 +41,14 @@ object AWSFilesChecker extends FilesChecker { override def countObjectsAtPath: String => Int = s3Client.countObjects(s3PrefixRegex) } + +object BlobFilesChecker extends FilesChecker { + import ObjectCounterInstances.blobObjectCounter + import ObjectCounterSyntax._ + + private lazy val containerClient = Operations.blobContainerClient + + // The root of the endpoint + container specified in reference.conf will be substituted for az:// + private val azurePrefixRange = "^az:\\/\\/.*" + override def countObjectsAtPath: String => Int = ObjectCounterSyntax(containerClient).countObjects(azurePrefixRange) +} diff --git a/centaur/src/main/scala/centaur/test/ObjectCounter.scala b/centaur/src/main/scala/centaur/test/ObjectCounter.scala index 46affc7d552..a5cb51ea190 100644 --- a/centaur/src/main/scala/centaur/test/ObjectCounter.scala +++ b/centaur/src/main/scala/centaur/test/ObjectCounter.scala @@ -1,5 +1,6 @@ package centaur.test +import com.azure.storage.blob.BlobContainerClient import com.google.cloud.storage.Storage.BlobListOption import com.google.cloud.storage.{Blob, Storage} import software.amazon.awssdk.services.s3.S3Client @@ -38,12 +39,31 @@ object ObjectCounterInstances { storage.list(g.bucket, BlobListOption.prefix(g.directory)).iterateAll.asScala listObjectsAtPath(_).size } + + implicit val blobObjectCounter: ObjectCounter[BlobContainerClient] = (containerClient: BlobContainerClient) => { + val pathToInt: Path => Int = providedPath => { + // Our path parsing is somewhat GCP centric. Convert to a blob path starting from the container root. + def pathToBlobPath(parsedPath: Path): String = + (Option(parsedPath.bucket), Option(parsedPath.directory)) match { + case (None, _) => "" + case (Some(_), None) => parsedPath.bucket + case (Some(_), Some(_)) => parsedPath.bucket + "/" + parsedPath.directory + } + + val fullPath = pathToBlobPath(providedPath) + val blobsInFolder = containerClient.listBlobsByHierarchy(fullPath) + // if something "isPrefix", it's a directory. Otherwise, its a file. We just want to count files. + blobsInFolder.asScala.count(!_.isPrefix) + } + pathToInt(_) + } } object ObjectCounterSyntax { implicit class ObjectCounterSyntax[A](client: A) { - def countObjects(regex: String)(implicit c: ObjectCounter[A]): String => Int = c.parsePath(regex) andThen c.countObjectsAtPath(client) + def countObjects(regex: String)(implicit c: ObjectCounter[A]): String => Int = + c.parsePath(regex) andThen c.countObjectsAtPath(client) } } diff --git a/centaur/src/main/scala/centaur/test/Test.scala b/centaur/src/main/scala/centaur/test/Test.scala index 470de959381..66a4655a107 100644 --- a/centaur/src/main/scala/centaur/test/Test.scala +++ b/centaur/src/main/scala/centaur/test/Test.scala @@ -10,6 +10,7 @@ import centaur.test.metadata.WorkflowFlatMetadata import centaur.test.metadata.WorkflowFlatMetadata._ import centaur.test.submit.SubmitHttpResponse import centaur.test.workflow.Workflow +import com.azure.storage.blob.BlobContainerClient import com.google.api.services.genomics.v2alpha1.{Genomics, GenomicsScopes} import com.google.api.services.storage.StorageScopes import com.google.auth.Credentials @@ -21,8 +22,20 @@ import com.typesafe.scalalogging.StrictLogging import common.validation.Validation._ import configs.syntax._ import cromwell.api.CromwellClient.UnsuccessfulRequestException -import cromwell.api.model.{CallCacheDiff, Failed, HashDifference, SubmittedWorkflow, Succeeded, TerminalStatus, WaasDescription, WorkflowId, WorkflowMetadata, WorkflowStatus} +import cromwell.api.model.{ + CallCacheDiff, + Failed, + HashDifference, + SubmittedWorkflow, + Succeeded, + TerminalStatus, + WaasDescription, + WorkflowId, + WorkflowMetadata, + WorkflowStatus +} import cromwell.cloudsupport.aws.AwsConfiguration +import cromwell.cloudsupport.azure.AzureUtils import cromwell.cloudsupport.gcp.GoogleConfiguration import cromwell.cloudsupport.gcp.auth.GoogleAuthMode import io.circe.parser._ @@ -51,32 +64,28 @@ sealed abstract class Test[A] { object Test { def successful[A](value: A): Test[A] = testMonad.pure(value) - def invalidTestDefinition[A](message: String, workflowDefinition: Workflow): Test[A] = { + def invalidTestDefinition[A](message: String, workflowDefinition: Workflow): Test[A] = new Test[A] { override def run: IO[Nothing] = IO.raiseError(CentaurTestException(message, workflowDefinition)) } - } implicit val testMonad: Monad[Test] = new Monad[Test] { - override def flatMap[A, B](fa: Test[A])(f: A => Test[B]): Test[B] = { + override def flatMap[A, B](fa: Test[A])(f: A => Test[B]): Test[B] = new Test[B] { override def run: IO[B] = fa.run flatMap { f(_).run } } - } - override def pure[A](x: A): Test[A] = { + override def pure[A](x: A): Test[A] = new Test[A] { override def run: IO[A] = IO.pure(x) } - } /** Call the default non-stack-safe but correct version of this method. */ - override def tailRecM[A, B](a: A)(f: A => Test[Either[A, B]]): Test[B] = { + override def tailRecM[A, B](a: A)(f: A => Test[Either[A, B]]): Test[B] = flatMap(f(a)) { case Right(b) => pure(b) case Left(nextA) => tailRecM(nextA)(f) } - } } implicit class TestableIO[A](a: IO[A]) { @@ -101,14 +110,14 @@ object Operations extends StrictLogging { lazy val authName: String = googleConf.getString("auth") lazy val genomicsEndpointUrl: String = googleConf.getString("genomics.endpoint-url") lazy val genomicsAndStorageScopes = List(StorageScopes.CLOUD_PLATFORM_READ_ONLY, GenomicsScopes.GENOMICS) - lazy val credentials: Credentials = configuration.auth(authName) + lazy val credentials: Credentials = configuration + .auth(authName) .unsafe .credentials(genomicsAndStorageScopes) - lazy val credentialsProjectOption: Option[String] = { - Option(credentials) collect { - case serviceAccountCredentials: ServiceAccountCredentials => serviceAccountCredentials.getProjectId + lazy val credentialsProjectOption: Option[String] = + Option(credentials) collect { case serviceAccountCredentials: ServiceAccountCredentials => + serviceAccountCredentials.getProjectId } - } lazy val confProjectOption: Option[String] = googleConf.get[Option[String]]("project") valueOrElse None // The project from the config or from the credentials. By default the project is read from the system environment. lazy val projectOption: Option[String] = confProjectOption orElse credentialsProjectOption @@ -138,25 +147,33 @@ object Operations extends StrictLogging { lazy val awsConfiguration: AwsConfiguration = AwsConfiguration(CentaurConfig.conf) lazy val awsConf: Config = CentaurConfig.conf.getConfig("aws") lazy val awsAuthName: String = awsConf.getString("auths") - lazy val region: String = awsConf.getString("region") - lazy val accessKeyId: String = awsConf.getString("access-key") + lazy val region: String = awsConf.getString("region") + lazy val accessKeyId: String = awsConf.getString("access-key") lazy val secretAccessKey: String = awsConf.getString("secret-key") def buildAmazonS3Client: S3Client = { val basicAWSCredentials = AwsBasicCredentials.create(accessKeyId, secretAccessKey) - S3Client.builder() + S3Client + .builder() .region(Region.of(region)) .credentialsProvider(StaticCredentialsProvider.create(basicAWSCredentials)) .build() } - def submitWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = { + lazy val azureConfig: Config = CentaurConfig.conf.getConfig("azure") + val azureSubscription = azureConfig.getString("subscription") + val blobContainer = azureConfig.getString("container") + val azureEndpoint = azureConfig.getString("endpoint") + // NB: Centaur will throw an exception if it isn't able to authenticate with Azure blob storage via the local environment. + lazy val blobContainerClient: BlobContainerClient = + AzureUtils.buildContainerClientFromLocalEnvironment(blobContainer, azureEndpoint, Option(azureSubscription)).get + + def submitWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = new Test[SubmittedWorkflow] { override def run: IO[SubmittedWorkflow] = for { id <- CentaurCromwellClient.submit(workflow) } yield id } - } /** * A smoke test of the version endpoint, this confirms that a) nothing explodes and b) the result must be a JSON object @@ -171,15 +188,15 @@ object Operations extends StrictLogging { def checkTimingRequirement(timeRequirement: Option[FiniteDuration]): Test[FiniteDuration] = new Test[FiniteDuration] { override def run: IO[FiniteDuration] = timeRequirement match { case Some(duration) => IO.pure(duration) - case None => IO.raiseError(new Exception("Duration value for 'maximumTime' required but not supplied in test config")) + case None => + IO.raiseError(new Exception("Duration value for 'maximumTime' required but not supplied in test config")) } } def checkFastEnough(before: Long, after: Long, allowance: FiniteDuration): Test[Unit] = new Test[Unit] { - override def run: IO[Unit] = { + override def run: IO[Unit] = if (after - before < allowance.toSeconds) IO.pure(()) else IO.raiseError(new Exception(s"Test took too long. Allowance was $allowance. Actual time: ${after - before}")) - } } def timingVerificationNotSupported(timingRequirement: Option[FiniteDuration]): Test[Unit] = new Test[Unit] { @@ -189,7 +206,7 @@ object Operations extends StrictLogging { } - def checkDescription(workflow: Workflow, validityExpectation: Option[Boolean], retries: Int = 3): Test[Unit] = { + def checkDescription(workflow: Workflow, validityExpectation: Option[Boolean], retries: Int = 3): Test[Unit] = new Test[Unit] { private val timeout = 60.seconds @@ -201,77 +218,79 @@ object Operations extends StrictLogging { case None => IO.pure(()) case Some(d.valid) => IO.pure(()) case Some(otherExpectation) => - logger.error(s"Unexpected 'valid=${d.valid}' response when expecting $otherExpectation. Full unexpected description:${System.lineSeparator()}$d") - IO.raiseError(new Exception(s"Expected this workflow's /describe validity to be '$otherExpectation' but got: '${d.valid}' (errors: ${d.errors.mkString(", ")})")) + logger.error( + s"Unexpected 'valid=${d.valid}' response when expecting $otherExpectation. Full unexpected description:${System + .lineSeparator()}$d" + ) + IO.raiseError( + new Exception( + s"Expected this workflow's /describe validity to be '$otherExpectation' but got: '${d.valid}' (errors: ${d.errors + .mkString(", ")})" + ) + ) } - }).timeoutTo(timeout, IO { - if (alreadyTried + 1 >= retries) { - throw new TimeoutException("Timeout from checkDescription 60 seconds: " + timeoutStackTraceString) - } else { - logger.warn(s"checkDescription timeout on attempt ${alreadyTried + 1}. ") - checkDescriptionInner(alreadyTried + 1) - () + }).timeoutTo( + timeout, + IO { + if (alreadyTried + 1 >= retries) { + throw new TimeoutException("Timeout from checkDescription 60 seconds: " + timeoutStackTraceString) + } else { + logger.warn(s"checkDescription timeout on attempt ${alreadyTried + 1}. ") + checkDescriptionInner(alreadyTried + 1) + () + } } - }) + ) } - - override def run: IO[Unit] = { - - + override def run: IO[Unit] = // We can't describe workflows based on zipped imports, so don't try: if (workflow.skipDescribeEndpointValidation || workflow.data.zippedImports.nonEmpty) { IO.pure(()) } else { checkDescriptionInner(0) } - } } - } - def submitInvalidWorkflow(workflow: Workflow): Test[SubmitHttpResponse] = { + def submitInvalidWorkflow(workflow: Workflow): Test[SubmitHttpResponse] = new Test[SubmitHttpResponse] { - override def run: IO[SubmitHttpResponse] = { - CentaurCromwellClient.submit(workflow).redeemWith( - { - case unsuccessfulRequestException: UnsuccessfulRequestException => - val httpResponse = unsuccessfulRequestException.httpResponse - val statusCode = httpResponse.status.intValue() - httpResponse.entity match { - case akka.http.scaladsl.model.HttpEntity.Strict(_, data) => - IO.pure(SubmitHttpResponse(statusCode, data.utf8String)) - case _ => - val message = s"Expected a strict http response entity but got ${httpResponse.entity}" - IO.raiseError(CentaurTestException(message, workflow, unsuccessfulRequestException)) - } - case unexpected: Exception => - val message = s"Unexpected error: ${unexpected.getMessage}" - IO.raiseError(CentaurTestException(message, workflow, unexpected)) - case throwable: Throwable => throw throwable - }, - { - submittedWorkflow => { + override def run: IO[SubmitHttpResponse] = + CentaurCromwellClient + .submit(workflow) + .redeemWith( + { + case unsuccessfulRequestException: UnsuccessfulRequestException => + val httpResponse = unsuccessfulRequestException.httpResponse + val statusCode = httpResponse.status.intValue() + httpResponse.entity match { + case akka.http.scaladsl.model.HttpEntity.Strict(_, data) => + IO.pure(SubmitHttpResponse(statusCode, data.utf8String)) + case _ => + val message = s"Expected a strict http response entity but got ${httpResponse.entity}" + IO.raiseError(CentaurTestException(message, workflow, unsuccessfulRequestException)) + } + case unexpected: Exception => + val message = s"Unexpected error: ${unexpected.getMessage}" + IO.raiseError(CentaurTestException(message, workflow, unexpected)) + case throwable: Throwable => throw throwable + }, + { submittedWorkflow => val message = s"Expected a failure but got a successfully submitted workflow with id ${submittedWorkflow.id}" IO.raiseError(CentaurTestException(message, workflow)) } - } - ) - } + ) } - } - def abortWorkflow(workflow: SubmittedWorkflow): Test[WorkflowStatus] = { + def abortWorkflow(workflow: SubmittedWorkflow): Test[WorkflowStatus] = new Test[WorkflowStatus] { override def run: IO[WorkflowStatus] = CentaurCromwellClient.abort(workflow) } - } - def waitFor(duration: FiniteDuration): Test[Unit] = { + def waitFor(duration: FiniteDuration): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = IO.sleep(duration) } - } /** * Polls until a valid status is reached. @@ -281,16 +300,18 @@ object Operations extends StrictLogging { def expectSomeProgress(workflow: SubmittedWorkflow, testDefinition: Workflow, expectedStatuses: Set[WorkflowStatus], - timeout: FiniteDuration): Test[SubmittedWorkflow] = { + timeout: FiniteDuration + ): Test[SubmittedWorkflow] = new Test[SubmittedWorkflow] { - def status(remainingTimeout: FiniteDuration): IO[SubmittedWorkflow] = { + def status(remainingTimeout: FiniteDuration): IO[SubmittedWorkflow] = for { workflowStatus <- CentaurCromwellClient.status(workflow) mappedStatus <- workflowStatus match { case s if expectedStatuses.contains(s) => IO.pure(workflow) case s: TerminalStatus => CentaurCromwellClient.metadata(workflow) flatMap { metadata => - val message = s"Unexpected terminal status $s while waiting for one of [${expectedStatuses.mkString(", ")}] (workflow ID: ${workflow.id})" + val message = + s"Unexpected terminal status $s while waiting for one of [${expectedStatuses.mkString(", ")}] (workflow ID: ${workflow.id})" IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata)) } case _ if remainingTimeout > 0.seconds => @@ -299,16 +320,15 @@ object Operations extends StrictLogging { s <- status(remainingTimeout - 10.seconds) } yield s case other => - val message = s"Cromwell failed to progress into any of the statuses [${expectedStatuses.mkString(", ")}]. Was still '$other' after $timeout (workflow ID: ${workflow.id})" + val message = + s"Cromwell failed to progress into any of the statuses [${expectedStatuses.mkString(", ")}]. Was still '$other' after $timeout (workflow ID: ${workflow.id})" IO.raiseError(CentaurTestException(message, testDefinition, workflow)) } } yield mappedStatus - } override def run: IO[SubmittedWorkflow] = status(timeout).timeout(CentaurConfig.maxWorkflowLength) } - } /** * Polls until a specific status is reached. @@ -317,9 +337,10 @@ object Operations extends StrictLogging { */ def pollUntilStatus(workflow: SubmittedWorkflow, testDefinition: Workflow, - expectedStatus: WorkflowStatus): Test[SubmittedWorkflow] = { + expectedStatus: WorkflowStatus + ): Test[SubmittedWorkflow] = new Test[SubmittedWorkflow] { - def status: IO[SubmittedWorkflow] = { + def status: IO[SubmittedWorkflow] = for { workflowStatus <- CentaurCromwellClient.status(workflow) mappedStatus <- workflowStatus match { @@ -327,35 +348,38 @@ object Operations extends StrictLogging { case s: TerminalStatus => val reducedMetadataOptions: Map[String, List[String]] = CentaurCromwellClient.defaultMetadataArgs.getOrElse(Map.empty) ++ Map( - "includeKey" -> (List("status") ++ (if (expectedStatus == Succeeded) List("failures") else List.empty)), + "includeKey" -> (List("status") ++ (if (expectedStatus == Succeeded) List("failures") + else List.empty)), "expandSubWorkflows" -> List("false") ) - CentaurCromwellClient.metadata(workflow = workflow, args = Option(reducedMetadataOptions)) flatMap { metadata => - val failuresString = if (expectedStatus == Succeeded) { - (for { - metadataJson <- parse(metadata.value).toOption - asObject <- metadataJson.asObject - failures <- asObject.toMap.get("failures") - } yield s" Metadata 'failures' content: ${failures.spaces2}").getOrElse("No additional failure information found in metadata.") - } else { - "" - } - - val message = s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id}).$failuresString" - IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata)) + CentaurCromwellClient.metadata(workflow = workflow, args = Option(reducedMetadataOptions)) flatMap { + metadata => + val failuresString = if (expectedStatus == Succeeded) { + (for { + metadataJson <- parse(metadata.value).toOption + asObject <- metadataJson.asObject + failures <- asObject.toMap.get("failures") + } yield s" Metadata 'failures' content: ${failures.spaces2}") + .getOrElse("No additional failure information found in metadata.") + } else { + "" + } + + val message = + s"Unexpected terminal status $s but was waiting for $expectedStatus (workflow ID: ${workflow.id}).$failuresString" + IO.raiseError(CentaurTestException(message, testDefinition, workflow, metadata)) } - case _ => for { - _ <- IO.sleep(10.seconds) - s <- status - } yield s + case _ => + for { + _ <- IO.sleep(10.seconds) + s <- status + } yield s } } yield mappedStatus - } override def run: IO[SubmittedWorkflow] = status.timeout(CentaurConfig.maxWorkflowLength) } - } /** * Validate that the given jobId matches the one in the metadata @@ -364,7 +388,8 @@ object Operations extends StrictLogging { workflow: SubmittedWorkflow, metadata: WorkflowMetadata, callFqn: String, - formerJobId: String): Test[Unit] = { + formerJobId: String + ): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = CentaurCromwellClient.metadata(workflow) flatMap { s => s.asFlat.value.get(s"calls.$callFqn.jobId") match { @@ -378,56 +403,58 @@ object Operations extends StrictLogging { } } } - } - def validatePAPIAborted(workflowDefinition: Workflow, workflow: SubmittedWorkflow, jobId: String): Test[Unit] = { + def validatePAPIAborted(workflowDefinition: Workflow, workflow: SubmittedWorkflow, jobId: String): Test[Unit] = new Test[Unit] { - def checkPAPIAborted(): IO[Unit] = { + def checkPAPIAborted(): IO[Unit] = for { - operation <- IO { genomics.projects().operations().get(jobId).execute() } + operation <- IO(genomics.projects().operations().get(jobId).execute()) done = operation.getDone operationError = Option(operation.getError) - aborted = operationError.exists(_.getCode == 1) && operationError.exists(_.getMessage.startsWith("Operation canceled")) - result <- if (!(done && aborted)) { - CentaurCromwellClient.metadata(workflow) flatMap { metadata => - val message = s"Underlying JES job was not aborted properly. " + - s"Done = $done. Error = ${operationError.map(_.getMessage).getOrElse("N/A")} (workflow ID: ${workflow.id})" - IO.raiseError(CentaurTestException(message, workflowDefinition, workflow, metadata)) - } - } else IO.unit + aborted = operationError.exists(_.getCode == 1) && operationError.exists( + _.getMessage.startsWith("Operation canceled") + ) + result <- + if (!(done && aborted)) { + CentaurCromwellClient.metadata(workflow) flatMap { metadata => + val message = s"Underlying JES job was not aborted properly. " + + s"Done = $done. Error = ${operationError.map(_.getMessage).getOrElse("N/A")} (workflow ID: ${workflow.id})" + IO.raiseError(CentaurTestException(message, workflowDefinition, workflow, metadata)) + } + } else IO.unit } yield result - } override def run: IO[Unit] = if (jobId.startsWith("operations/")) { checkPAPIAborted() } else IO.unit } - } /** * Polls until a specific call is in Running state. Returns the job id. */ - def pollUntilCallIsRunning(workflowDefinition: Workflow, workflow: SubmittedWorkflow, callFqn: String): Test[String] = { + def pollUntilCallIsRunning(workflowDefinition: Workflow, + workflow: SubmittedWorkflow, + callFqn: String + ): Test[String] = { // Special case for sub workflow testing - def findJobIdInSubWorkflow(subWorkflowId: String): IO[Option[String]] = { + def findJobIdInSubWorkflow(subWorkflowId: String): IO[Option[String]] = for { metadata <- CentaurCromwellClient .metadataWithId(WorkflowId.fromString(subWorkflowId)) .redeem(_ => None, Option.apply) jobId <- IO.pure(metadata.flatMap(_.asFlat.value.get("calls.inner_abort.aborted.jobId"))) } yield jobId.map(_.asInstanceOf[JsString].value) - } - def valueAsString(key: String, metadata: WorkflowMetadata) = { + def valueAsString(key: String, metadata: WorkflowMetadata) = metadata.asFlat.value.get(key).map(_.asInstanceOf[JsString].value) - } def findCallStatus(metadata: WorkflowMetadata): IO[Option[(String, String)]] = { val status = metadata.asFlat.value.get(s"calls.$callFqn.executionStatus") val statusString = status.map(_.asInstanceOf[JsString].value) for { - jobId <- valueAsString(s"calls.$callFqn.jobId", metadata).map(jobId => IO.pure(Option(jobId))) + jobId <- valueAsString(s"calls.$callFqn.jobId", metadata) + .map(jobId => IO.pure(Option(jobId))) .orElse(valueAsString(s"calls.$callFqn.subWorkflowId", metadata).map(findJobIdInSubWorkflow)) .getOrElse(IO.pure(None)) pair = (statusString, jobId) match { @@ -438,7 +465,7 @@ object Operations extends StrictLogging { } new Test[String] { - def doPerform(): IO[String] = { + def doPerform(): IO[String] = for { // We don't want to keep going forever if the workflow failed status <- CentaurCromwellClient.status(workflow) @@ -455,13 +482,13 @@ object Operations extends StrictLogging { case Some(("Failed", _)) => val message = s"$callFqn failed" IO.raiseError(CentaurTestException(message, workflowDefinition, workflow, metadata)) - case _ => for { - _ <- IO.sleep(5.seconds) - recurse <- doPerform() - } yield recurse + case _ => + for { + _ <- IO.sleep(5.seconds) + recurse <- doPerform() + } yield recurse } } yield result - } override def run: IO[String] = doPerform().timeout(CentaurConfig.maxWorkflowLength) } @@ -474,34 +501,40 @@ object Operations extends StrictLogging { for { md <- CentaurCromwellClient.metadata(workflowB) - calls = md.asFlat.value.keySet.flatMap({ + calls = md.asFlat.value.keySet.flatMap { case callNameRegexp(name) => Option(name) case _ => None - }) - diffs <- calls.toList.traverse[IO, CallCacheDiff]({ callName => + } + diffs <- calls.toList.traverse[IO, CallCacheDiff] { callName => CentaurCromwellClient.callCacheDiff(workflowA, callName, workflowB, callName) - }) + } } yield diffs.flatMap(_.hashDifferential) } - override def run: IO[Unit] = { + override def run: IO[Unit] = hashDiffOfAllCalls map { case diffs if diffs.nonEmpty && CentaurCromwellClient.LogFailures => Console.err.println(s"Hash differential for ${workflowA.id} and ${workflowB.id}") - diffs.map({ diff => - s"For key ${diff.hashKey}:\nCall A: ${diff.callA.getOrElse("N/A")}\nCall B: ${diff.callB.getOrElse("N/A")}" - }).foreach(Console.err.println) + diffs + .map { diff => + s"For key ${diff.hashKey}:\nCall A: ${diff.callA.getOrElse("N/A")}\nCall B: ${diff.callB.getOrElse("N/A")}" + } + .foreach(Console.err.println) case _ => } - } } /* Select only those flat metadata items whose keys begin with the specified prefix, removing the prefix from the keys. Also * perform variable substitutions for UUID and WORKFLOW_ROOT and remove any ~> Centaur metadata expectation metacharacters. */ - private def selectMetadataExpectationSubsetByPrefix(workflow: Workflow, prefix: String, workflowId: WorkflowId, workflowRoot: String): List[(String, JsValue)] = { + private def selectMetadataExpectationSubsetByPrefix(workflow: Workflow, + prefix: String, + workflowId: WorkflowId, + workflowRoot: String + ): List[(String, JsValue)] = { import WorkflowFlatMetadata._ def replaceVariables(value: JsValue): JsValue = value match { - case s: JsString => JsString(s.value.replaceExpectationVariables(workflowId, workflowRoot).replaceFirst("^~>", "")) + case s: JsString => + JsString(s.value.replaceExpectationVariables(workflowId, workflowRoot).replaceFirst("^~>", "")) case o => o } val filterLabels: PartialFunction[(String, JsValue), (String, JsValue)] = { @@ -516,7 +549,8 @@ object Operations extends StrictLogging { def fetchAndValidateOutputs(submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - workflowRoot: String): Test[JsObject] = new Test[JsObject] { + workflowRoot: String + ): Test[JsObject] = new Test[JsObject] { def checkOutputs(expectedOutputs: List[(String, JsValue)])(actualOutputs: Map[String, JsValue]): IO[Unit] = { val expected = expectedOutputs.toSet @@ -526,11 +560,13 @@ object Operations extends StrictLogging { lazy val inExpectedButNotInActual = expected.diff(actual) if (!workflow.allowOtherOutputs && inActualButNotInExpected.nonEmpty) { - val message = s"In actual outputs but not in expected and other outputs not allowed: ${inActualButNotInExpected.mkString(", ")}" + val message = + s"In actual outputs but not in expected and other outputs not allowed: ${inActualButNotInExpected.mkString(", ")}" IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) } else if (inExpectedButNotInActual.nonEmpty) { - val message = s"In actual outputs but not in expected: ${inExpectedButNotInActual.mkString(", ")}" + System.lineSeparator + - s"In expected outputs but not in actual: ${inExpectedButNotInActual.mkString(", ")}" + val message = + s"In actual outputs but not in expected: ${inExpectedButNotInActual.mkString(", ")}" + System.lineSeparator + + s"In expected outputs but not in actual: ${inExpectedButNotInActual.mkString(", ")}" IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) } else { IO.unit @@ -540,7 +576,8 @@ object Operations extends StrictLogging { override def run: IO[JsObject] = { import centaur.test.metadata.WorkflowFlatOutputs._ - val expectedOutputs: List[(String, JsValue)] = selectMetadataExpectationSubsetByPrefix(workflow, "outputs.", submittedWorkflow.id, workflowRoot) + val expectedOutputs: List[(String, JsValue)] = + selectMetadataExpectationSubsetByPrefix(workflow, "outputs.", submittedWorkflow.id, workflowRoot) for { outputs <- CentaurCromwellClient.outputs(submittedWorkflow) @@ -553,7 +590,8 @@ object Operations extends StrictLogging { def fetchAndValidateLabels(submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - workflowRoot: String): Test[Unit] = new Test[Unit] { + workflowRoot: String + ): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = { import centaur.test.metadata.WorkflowFlatLabels._ @@ -561,17 +599,18 @@ object Operations extends StrictLogging { val expectedLabels: List[(String, JsValue)] = workflowIdLabel :: selectMetadataExpectationSubsetByPrefix(workflow, "labels.", submittedWorkflow.id, workflowRoot) - def validateLabels(actualLabels: Map[String, JsValue]) = { val diff = expectedLabels.toSet.diff(actualLabels.toSet) if (diff.isEmpty) { IO.unit } else { - IO.raiseError(CentaurTestException( - s"In expected labels but not in actual: ${diff.mkString(", ")}", - workflow, - submittedWorkflow - )) + IO.raiseError( + CentaurTestException( + s"In expected labels but not in actual: ${diff.mkString(", ")}", + workflow, + submittedWorkflow + ) + ) } } @@ -584,75 +623,77 @@ object Operations extends StrictLogging { } /** Compares logs filtered from the raw `metadata` endpoint with the `logs` endpoint. */ - def validateLogs(metadata: WorkflowMetadata, - submittedWorkflow: SubmittedWorkflow, - workflow: Workflow): Test[Unit] = new Test[Unit] { - val suffixes = Set("stdout", "shardIndex", "stderr", "attempt", "backendLogs.log") - - def removeSubworkflowKeys(flattened: Map[String, JsValue]): Map[String, JsValue] = { - val subWorkflowIdPrefixes = flattened.keys.filter(_.endsWith(".subWorkflowId")).map(s => s.substring(0, s.lastIndexOf('.'))) - flattened filter { case (k, _) => !subWorkflowIdPrefixes.exists(k.startsWith) } - } - - // Filter to only include the fields in the flattened metadata that should appear in the logs endpoint. - def filterForLogsFields(flattened: Map[String, JsValue]): Map[String, JsValue] = removeSubworkflowKeys(flattened).filter { - case (k, _) => k == "id" || suffixes.exists(s => k.endsWith("." + s) && !k.contains(".outputs.") && !k.startsWith("outputs.")) - } + def validateLogs(metadata: WorkflowMetadata, submittedWorkflow: SubmittedWorkflow, workflow: Workflow): Test[Unit] = + new Test[Unit] { + val suffixes = Set("stdout", "shardIndex", "stderr", "attempt", "backendLogs.log") - override def run: IO[Unit] = { + def removeSubworkflowKeys(flattened: Map[String, JsValue]): Map[String, JsValue] = { + val subWorkflowIdPrefixes = + flattened.keys.filter(_.endsWith(".subWorkflowId")).map(s => s.substring(0, s.lastIndexOf('.'))) + flattened filter { case (k, _) => !subWorkflowIdPrefixes.exists(k.startsWith) } + } - def validateLogsMetadata(flatLogs: Map[String, JsValue], flatFilteredMetadata: Map[String, JsValue]): IO[Unit] = - if (flatLogs.equals(flatFilteredMetadata)) { - IO.unit - } else { - val message = (List("actual logs endpoint output did not equal filtered metadata", "flat logs: ") ++ - flatLogs.toList ++ List("flat filtered metadata: ") ++ flatFilteredMetadata.toList).mkString("\n") - IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) + // Filter to only include the fields in the flattened metadata that should appear in the logs endpoint. + def filterForLogsFields(flattened: Map[String, JsValue]): Map[String, JsValue] = + removeSubworkflowKeys(flattened).filter { case (k, _) => + k == "id" || suffixes.exists(s => + k.endsWith("." + s) && !k.contains(".outputs.") && !k.startsWith("outputs.") + ) } - for { - logs <- CentaurCromwellClient.logs(submittedWorkflow) - flatLogs = logs.asFlat.value - flatFilteredMetadata = metadata.asFlat.value |> filterForLogsFields - _ <- validateLogsMetadata(flatLogs, flatFilteredMetadata) - } yield () + override def run: IO[Unit] = { + + def validateLogsMetadata(flatLogs: Map[String, JsValue], flatFilteredMetadata: Map[String, JsValue]): IO[Unit] = + if (flatLogs.equals(flatFilteredMetadata)) { + IO.unit + } else { + val message = (List("actual logs endpoint output did not equal filtered metadata", "flat logs: ") ++ + flatLogs.toList ++ List("flat filtered metadata: ") ++ flatFilteredMetadata.toList).mkString("\n") + IO.raiseError(CentaurTestException(message, workflow, submittedWorkflow)) + } + + for { + logs <- CentaurCromwellClient.logs(submittedWorkflow) + flatLogs = logs.asFlat.value + flatFilteredMetadata = metadata.asFlat.value |> filterForLogsFields + _ <- validateLogsMetadata(flatLogs, flatFilteredMetadata) + } yield () + } } - } - def fetchMetadata(submittedWorkflow: SubmittedWorkflow, - expandSubworkflows: Boolean): IO[WorkflowMetadata] = { + def fetchMetadata(submittedWorkflow: SubmittedWorkflow, expandSubworkflows: Boolean): IO[WorkflowMetadata] = CentaurCromwellClient.metadata(submittedWorkflow, expandSubworkflows = expandSubworkflows) - } def fetchAndValidateNonSubworkflowMetadata(submittedWorkflow: SubmittedWorkflow, workflowSpec: Workflow, - cacheHitUUID: Option[UUID] = None): Test[WorkflowMetadata] = { + cacheHitUUID: Option[UUID] = None + ): Test[WorkflowMetadata] = new Test[WorkflowMetadata] { def fetchOnce(): IO[WorkflowMetadata] = fetchMetadata(submittedWorkflow, expandSubworkflows = false) def eventuallyMetadata(workflow: SubmittedWorkflow, - expectedMetadata: WorkflowFlatMetadata): IO[WorkflowMetadata] = { - validateMetadata(workflow, expectedMetadata).handleErrorWith({ _ => + expectedMetadata: WorkflowFlatMetadata + ): IO[WorkflowMetadata] = + validateMetadata(workflow, expectedMetadata).handleErrorWith { _ => for { _ <- IO.sleep(2.seconds) recurse <- eventuallyMetadata(workflow, expectedMetadata) } yield recurse - }) - } + } def validateMetadata(workflow: SubmittedWorkflow, - expectedMetadata: WorkflowFlatMetadata): IO[WorkflowMetadata] = { - def checkDiff(diffs: Iterable[String], actualMetadata: WorkflowMetadata): IO[Unit] = { + expectedMetadata: WorkflowFlatMetadata + ): IO[WorkflowMetadata] = { + def checkDiff(diffs: Iterable[String], actualMetadata: WorkflowMetadata): IO[Unit] = if (diffs.nonEmpty) { val message = s"Invalid metadata response:\n -${diffs.mkString("\n -")}\n" IO.raiseError(CentaurTestException(message, workflowSpec, workflow, actualMetadata)) } else { IO.unit } - } - def validateUnwantedMetadata(actualMetadata: WorkflowMetadata): IO[Unit] = { + def validateUnwantedMetadata(actualMetadata: WorkflowMetadata): IO[Unit] = if (workflowSpec.notInMetadata.nonEmpty) { // Check that none of the "notInMetadata" keys are in the actual metadata val absentMdIntersect = workflowSpec.notInMetadata.toSet.intersect(actualMetadata.asFlat.value.keySet) @@ -665,23 +706,23 @@ object Operations extends StrictLogging { } else { IO.unit } - } - def validateAllowOtherOutputs(actualMetadata: WorkflowMetadata): IO[Unit] = { + def validateAllowOtherOutputs(actualMetadata: WorkflowMetadata): IO[Unit] = if (workflowSpec.allowOtherOutputs) IO.unit else { val flat = actualMetadata.asFlat.value val actualOutputs: Iterable[String] = flat.keys.filter(_.startsWith("outputs.")) - val expectedOutputs: Iterable[String] = workflowSpec.metadata.map(w => w.value.keys.filter(_.startsWith("outputs."))).getOrElse(List.empty) + val expectedOutputs: Iterable[String] = + workflowSpec.metadata.map(w => w.value.keys.filter(_.startsWith("outputs."))).getOrElse(List.empty) val diff = actualOutputs.toSet.diff(expectedOutputs.toSet) if (diff.nonEmpty) { - val message = s"Found unwanted keys in metadata with `allow-other-outputs` = false: ${diff.mkString(", ")}" + val message = + s"Found unwanted keys in metadata with `allow-other-outputs` = false: ${diff.mkString(", ")}" IO.raiseError(CentaurTestException(message, workflowSpec, workflow, actualMetadata)) } else { IO.unit } } - } for { actualMetadata <- fetchOnce() @@ -700,14 +741,14 @@ object Operations extends StrictLogging { case None => fetchOnce() } } - } def validateMetadataJson(testType: String, expected: JsObject, actual: JsObject, submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - allowableAddedOneWordFields: List[String]): IO[Unit] = { + allowableAddedOneWordFields: List[String] + ): IO[Unit] = if (actual.equals(expected)) { IO.unit } else { @@ -739,46 +780,66 @@ object Operations extends StrictLogging { } else { val writer: JsonWriter[Vector[Operation[JsValue]]] = new JsonWriter[Vector[Operation[JsValue]]] { def processOperation(op: Operation[JsValue]): JsValue = op match { - case Add(path, value) => JsObject(Map[String, JsValue]( - "description" -> JsString("Unexpected value found"), - "path" -> JsString(path.toString), - "value" -> value)) - case Copy(from, path) => JsObject(Map[String, JsValue]( - "description" -> JsString("Value(s) unexpectedly copied"), - "expected_at" -> JsString(from.toString), - "also_at" -> JsString(path.toString))) - case Move(from, path) => JsObject(Map[String, JsValue]( - "description" -> JsString("Value(s) unexpectedly moved"), - "expected_location" -> JsString(from.toString), - "actual_location" -> JsString(path.toString))) - case Remove(path, old) => JsObject(Map[String, JsValue]( - "description" -> JsString("Value missing"), - "expected_location" -> JsString(path.toString)) ++ - old.map(o => "expected_value" -> o)) - case Replace(path, value, old) => JsObject(Map[String, JsValue]( - "description" -> JsString("Incorrect value found"), - "path" -> JsString(path.toString), - "found_value" -> value) ++ old.map(o => "expected_value" -> o)) - case diffson.jsonpatch.Test(path, value) => JsObject(Map[String, JsValue]( - "op" -> JsString("test"), - "path" -> JsString(path.toString), - "value" -> value)) + case Add(path, value) => + JsObject( + Map[String, JsValue]("description" -> JsString("Unexpected value found"), + "path" -> JsString(path.toString), + "value" -> value + ) + ) + case Copy(from, path) => + JsObject( + Map[String, JsValue]("description" -> JsString("Value(s) unexpectedly copied"), + "expected_at" -> JsString(from.toString), + "also_at" -> JsString(path.toString) + ) + ) + case Move(from, path) => + JsObject( + Map[String, JsValue]("description" -> JsString("Value(s) unexpectedly moved"), + "expected_location" -> JsString(from.toString), + "actual_location" -> JsString(path.toString) + ) + ) + case Remove(path, old) => + JsObject( + Map[String, JsValue]("description" -> JsString("Value missing"), + "expected_location" -> JsString(path.toString) + ) ++ + old.map(o => "expected_value" -> o) + ) + case Replace(path, value, old) => + JsObject( + Map[String, JsValue]("description" -> JsString("Incorrect value found"), + "path" -> JsString(path.toString), + "found_value" -> value + ) ++ old.map(o => "expected_value" -> o) + ) + case diffson.jsonpatch.Test(path, value) => + JsObject( + Map[String, JsValue]("op" -> JsString("test"), "path" -> JsString(path.toString), "value" -> value) + ) } - override def write(vector: Vector[Operation[JsValue]]): JsValue = { + override def write(vector: Vector[Operation[JsValue]]): JsValue = JsArray(vector.map(processOperation)) - } } val jsonDiff = filteredDifferences.toJson(writer).prettyPrint - IO.raiseError(CentaurTestException(s"Error during $testType metadata comparison. Diff: $jsonDiff Expected: $expected Actual: $actual", workflow, submittedWorkflow)) + IO.raiseError( + CentaurTestException( + s"Error during $testType metadata comparison. Diff: $jsonDiff Expected: $expected Actual: $actual", + workflow, + submittedWorkflow + ) + ) } } - } def fetchAndValidateJobManagerStyleMetadata(submittedWorkflow: SubmittedWorkflow, workflow: Workflow, - prefetchedOriginalNonSubWorkflowMetadata: Option[String]): Test[WorkflowMetadata] = new Test[WorkflowMetadata] { + prefetchedOriginalNonSubWorkflowMetadata: Option[String] + ): Test[WorkflowMetadata] = new Test[WorkflowMetadata] { // If the non-subworkflow metadata was already fetched, there's no need to fetch it again. def originalMetadataStringIO: IO[String] = prefetchedOriginalNonSubWorkflowMetadata match { @@ -790,19 +851,43 @@ object Operations extends StrictLogging { originalMetadata <- originalMetadataStringIO jmMetadata <- CentaurCromwellClient.metadata( workflow = submittedWorkflow, - Option(CentaurCromwellClient.defaultMetadataArgs.getOrElse(Map.empty) ++ jmArgs)) + Option(CentaurCromwellClient.defaultMetadataArgs.getOrElse(Map.empty) ++ jmArgs) + ) jmMetadataObject <- IO.fromTry(Try(jmMetadata.value.parseJson.asJsObject)) expectation <- IO.fromTry(Try(extractJmStyleMetadataFields(originalMetadata.parseJson.asJsObject))) - _ <- validateMetadataJson(testType = s"fetchAndValidateJobManagerStyleMetadata", expectation, jmMetadataObject, submittedWorkflow, workflow, allowableOneWordAdditionsInJmMetadata) + _ <- validateMetadataJson(testType = s"fetchAndValidateJobManagerStyleMetadata", + expectation, + jmMetadataObject, + submittedWorkflow, + workflow, + allowableOneWordAdditionsInJmMetadata + ) } yield jmMetadata } val oneWordJmIncludeKeys = List( - "attempt", "callRoot", "end", - "executionStatus", "failures", "inputs", "jobId", - "calls", "outputs", "shardIndex", "start", "stderr", "stdout", - "description", "executionEvents", "labels", "parentWorkflowId", - "returnCode", "status", "submission", "subWorkflowId", "workflowName" + "attempt", + "callRoot", + "end", + "executionStatus", + "failures", + "inputs", + "jobId", + "calls", + "outputs", + "shardIndex", + "start", + "stderr", + "stdout", + "description", + "executionEvents", + "labels", + "parentWorkflowId", + "returnCode", + "status", + "submission", + "subWorkflowId", + "workflowName" ) // Our Job Manager metadata validation works by comparing the first pull of metadata which satisfies all test requirements @@ -830,12 +915,13 @@ object Operations extends StrictLogging { // NB: this filter to remove "calls" is because - although it is a single word in the JM request, // it gets treated specially by the API (so has to be treated specially here too) - def processOneWordIncludes(json: JsObject) = (oneWordJmIncludeKeys.filterNot(_ == "calls") :+ "id").foldRight(JsObject.empty) { (toInclude, current) => - json.fields.get(toInclude) match { - case Some(jsonToInclude) => JsObject(current.fields + (toInclude -> jsonToInclude)) - case None => current + def processOneWordIncludes(json: JsObject) = + (oneWordJmIncludeKeys.filterNot(_ == "calls") :+ "id").foldRight(JsObject.empty) { (toInclude, current) => + json.fields.get(toInclude) match { + case Some(jsonToInclude) => JsObject(current.fields + (toInclude -> jsonToInclude)) + case None => current + } } - } def processCallCacheField(callJson: JsObject) = for { originalCallCachingField <- callJson.fields.get("callCaching") @@ -855,7 +941,9 @@ object Operations extends StrictLogging { } val workflowLevelWithOneWordIncludes = processOneWordIncludes(originalWorkflowMetadataJson) - val callsField = originalCallMetadataJson map { calls => Map("calls" -> processCallsSection(calls)) } getOrElse Map.empty + val callsField = originalCallMetadataJson map { calls => + Map("calls" -> processCallsSection(calls)) + } getOrElse Map.empty JsObject(workflowLevelWithOneWordIncludes.fields ++ callsField) } @@ -866,7 +954,8 @@ object Operations extends StrictLogging { def validateCacheResultField(workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow, metadata: WorkflowMetadata, - blacklistedValue: String): Test[Unit] = { + blacklistedValue: String + ): Test[Unit] = new Test[Unit] { override def run: IO[Unit] = { val badCacheResults = metadata.asFlat.value collect { @@ -882,24 +971,24 @@ object Operations extends StrictLogging { } } } - } def validateDirectoryContentsCounts(workflowDefinition: Workflow, submittedWorkflow: SubmittedWorkflow, - metadata: WorkflowMetadata): Test[Unit] = new Test[Unit] { + metadata: WorkflowMetadata + ): Test[Unit] = new Test[Unit] { private val workflowId = submittedWorkflow.id.id.toString override def run: IO[Unit] = workflowDefinition.directoryContentCounts match { case None => IO.unit case Some(directoryContentCountCheck) => - val counts = directoryContentCountCheck.expectedDirectoryContentsCounts map { - case (directory, count) => - val substitutedDir = directory.replaceAll("<>", workflowId) - (substitutedDir, count, directoryContentCountCheck.checkFiles.countObjectsAtPath(substitutedDir)) + val counts = directoryContentCountCheck.expectedDirectoryContentsCounts map { case (directory, count) => + val substitutedDir = directory.replaceAll("<>", workflowId) + (substitutedDir, count, directoryContentCountCheck.checkFiles.countObjectsAtPath(substitutedDir)) } val badCounts = counts collect { - case (directory, expectedCount, actualCount) if expectedCount != actualCount => s"Expected to find $expectedCount item(s) at $directory but got $actualCount" + case (directory, expectedCount, actualCount) if expectedCount != actualCount => + s"Expected to find $expectedCount item(s) at $directory but got $actualCount" } if (badCounts.isEmpty) { IO.unit @@ -912,21 +1001,22 @@ object Operations extends StrictLogging { def validateNoCacheHits(submittedWorkflow: SubmittedWorkflow, metadata: WorkflowMetadata, - workflowDefinition: Workflow): Test[Unit] = { + workflowDefinition: Workflow + ): Test[Unit] = validateCacheResultField(workflowDefinition, submittedWorkflow, metadata, "Cache Hit") - } def validateNoCacheMisses(submittedWorkflow: SubmittedWorkflow, metadata: WorkflowMetadata, - workflowDefinition: Workflow): Test[Unit] = { + workflowDefinition: Workflow + ): Test[Unit] = validateCacheResultField(workflowDefinition, submittedWorkflow, metadata, "Cache Miss") - } def validateSubmitFailure(workflow: Workflow, expectedSubmitResponse: SubmitHttpResponse, - actualSubmitResponse: SubmitHttpResponse): Test[Unit] = { + actualSubmitResponse: SubmitHttpResponse + ): Test[Unit] = new Test[Unit] { - override def run: IO[Unit] = { + override def run: IO[Unit] = if (expectedSubmitResponse == actualSubmitResponse) { IO.unit } else { @@ -940,7 +1030,5 @@ object Operations extends StrictLogging { |""".stripMargin IO.raiseError(CentaurTestException(message, workflow)) } - } } - } } diff --git a/centaur/src/main/scala/centaur/test/TestOptions.scala b/centaur/src/main/scala/centaur/test/TestOptions.scala index 10354c756bb..ceba238a4da 100644 --- a/centaur/src/main/scala/centaur/test/TestOptions.scala +++ b/centaur/src/main/scala/centaur/test/TestOptions.scala @@ -18,15 +18,13 @@ object TestOptions { Apply[ErrorOr].map2(tags, ignore)((t, i) => TestOptions(t, i)) } - def tagsFromConfig(conf: Config): ErrorOr[List[String]] = { + def tagsFromConfig(conf: Config): ErrorOr[List[String]] = conf.get[List[String]]("tags") match { case Success(tagStrings) => Valid(tagStrings.map(_.toLowerCase).distinct) case Failure(_) => Valid(List.empty[String]) } - } - - def ignoreFromConfig(conf: Config): ErrorOr[Boolean] = { + def ignoreFromConfig(conf: Config): ErrorOr[Boolean] = if (conf.hasPath("ignore")) { conf.get[Boolean]("ignore") match { case Success(ignore) => Valid(ignore) @@ -35,6 +33,4 @@ object TestOptions { } else { Valid(false) } - } } - diff --git a/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala b/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala index 774f852237d..6c006283894 100644 --- a/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala +++ b/centaur/src/main/scala/centaur/test/formulas/TestFormulas.scala @@ -12,11 +12,21 @@ import centaur.test.workflow.Workflow import centaur.test.{Operations, Test} import centaur.{CentaurConfig, CromwellManager, CromwellTracker, ManagedCromwellServer} import com.typesafe.scalalogging.StrictLogging -import cromwell.api.model.{Aborted, Aborting, Failed, Running, SubmittedWorkflow, Succeeded, TerminalStatus, WorkflowMetadata} +import cromwell.api.model.{ + Aborted, + Aborting, + Failed, + Running, + SubmittedWorkflow, + Succeeded, + TerminalStatus, + WorkflowMetadata +} import scala.concurrent.duration._ import centaur.test.metadata.WorkflowFlatMetadata._ import spray.json.JsString + /** * A collection of test formulas which can be used, building upon operations by chaining them together via a * for comprehension. These assembled formulas can then be run by a client @@ -33,7 +43,7 @@ object TestFormulas extends StrictLogging { |""".stripMargin.trim ) - private def runWorkflowUntilTerminalStatus(workflow: Workflow, status: TerminalStatus): Test[SubmittedWorkflow] = { + private def runWorkflowUntilTerminalStatus(workflow: Workflow, status: TerminalStatus): Test[SubmittedWorkflow] = for { _ <- checkVersion() s <- submitWorkflow(workflow) @@ -41,14 +51,15 @@ object TestFormulas extends StrictLogging { workflow = s, testDefinition = workflow, expectedStatuses = Set(Running, status), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(s, workflow, status) } yield s - } - private def runSuccessfulWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = runWorkflowUntilTerminalStatus(workflow, Succeeded) - private def runFailingWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = runWorkflowUntilTerminalStatus(workflow, Failed) + private def runSuccessfulWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = + runWorkflowUntilTerminalStatus(workflow, Succeeded) + private def runFailingWorkflow(workflow: Workflow): Test[SubmittedWorkflow] = + runWorkflowUntilTerminalStatus(workflow, Failed) def runSuccessfulWorkflowAndVerifyTimeAndOutputs(workflowDefinition: Workflow): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) @@ -56,18 +67,28 @@ object TestFormulas extends StrictLogging { beforeTimestamp = OffsetDateTime.now().toInstant.getEpochSecond submittedWorkflow <- runSuccessfulWorkflow(workflowDefinition) afterTimestamp = OffsetDateTime.now().toInstant.getEpochSecond - _ <- fetchAndValidateOutputs(submittedWorkflow, workflowDefinition, "ROOT NOT SUPPORTED IN TIMING/OUTPUT ONLY TESTS") + _ <- fetchAndValidateOutputs(submittedWorkflow, + workflowDefinition, + "ROOT NOT SUPPORTED IN TIMING/OUTPUT ONLY TESTS" + ) _ <- checkFastEnough(beforeTimestamp, afterTimestamp, timeAllowance) } yield SubmitResponse(submittedWorkflow) - def runSuccessfulWorkflowAndVerifyMetadata(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { + def runSuccessfulWorkflowAndVerifyMetadata( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) submittedWorkflow <- runSuccessfulWorkflow(workflowDefinition) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value)) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value) + ) notArchivedFlatMetadata = metadata.asFlat - workflowRoot = notArchivedFlatMetadata.value.get("workflowRoot").collectFirst { case JsString(r) => r } getOrElse "No Workflow Root" + workflowRoot = notArchivedFlatMetadata.value.get("workflowRoot").collectFirst { case JsString(r) => + r + } getOrElse "No Workflow Root" _ <- fetchAndValidateOutputs(submittedWorkflow, workflowDefinition, workflowRoot) _ <- fetchAndValidateLabels(submittedWorkflow, workflowDefinition, workflowRoot) _ <- validateLogs(metadata, submittedWorkflow, workflowDefinition) @@ -75,17 +96,24 @@ object TestFormulas extends StrictLogging { _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) - def runFailingWorkflowAndVerifyMetadata(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { + def runFailingWorkflowAndVerifyMetadata( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = None) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) submittedWorkflow <- runFailingWorkflow(workflowDefinition) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value)) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = Option(metadata.value) + ) _ = cromwellTracker.track(metadata) _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) - def runWorkflowTwiceExpectingCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runWorkflowTwiceExpectingCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) @@ -93,14 +121,18 @@ object TestFormulas extends StrictLogging { secondWf <- runSuccessfulWorkflow(workflowDefinition.secondRun) _ <- printHashDifferential(firstWF, secondWf) metadata <- fetchAndValidateNonSubworkflowMetadata(secondWf, workflowDefinition, Option(firstWF.id.id)) - _ <- fetchAndValidateJobManagerStyleMetadata(secondWf, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(secondWf, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheMisses(secondWf, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, secondWf, metadata) } yield SubmitResponse(secondWf) - } - def runWorkflowThriceExpectingCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runWorkflowThriceExpectingCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) @@ -115,41 +147,48 @@ object TestFormulas extends StrictLogging { _ <- validateNoCacheMisses(thirdWf, metadataThree, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, thirdWf, metadataThree) } yield SubmitResponse(thirdWf) - } - def runWorkflowTwiceExpectingNoCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runWorkflowTwiceExpectingNoCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) _ <- runSuccessfulWorkflow(workflowDefinition) // Build caches testWf <- runSuccessfulWorkflow(workflowDefinition.secondRun) metadata <- fetchAndValidateNonSubworkflowMetadata(testWf, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(testWf, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(testWf, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheHits(testWf, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, testWf, metadata) } yield SubmitResponse(testWf) - } - def runFailingWorkflowTwiceExpectingNoCaching(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def runFailingWorkflowTwiceExpectingNoCaching( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = None) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) _ <- runFailingWorkflow(workflowDefinition) // Build caches testWf <- runFailingWorkflow(workflowDefinition) metadata <- fetchAndValidateNonSubworkflowMetadata(testWf, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(testWf, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(testWf, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheHits(testWf, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, testWf, metadata) } yield SubmitResponse(testWf) - } private def cromwellRestart(workflowDefinition: Workflow, callMarker: CallMarker, testRecover: Boolean, - finalStatus: TerminalStatus)( - implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + finalStatus: TerminalStatus + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = CentaurConfig.runMode match { case ManagedCromwellServer(_, postRestart, withRestart) if withRestart => for { @@ -163,27 +202,31 @@ object TestFormulas extends StrictLogging { workflow = submittedWorkflow, testDefinition = workflowDefinition, expectedStatuses = Set(Running, finalStatus), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(submittedWorkflow, workflowDefinition, finalStatus) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) - _ <- if (testRecover) { - validateRecovered(workflowDefinition, submittedWorkflow, metadata, callMarker.callKey, jobId) - } - else { - Test.successful(()) - } + _ <- + if (testRecover) { + validateRecovered(workflowDefinition, submittedWorkflow, metadata, callMarker.callKey, jobId) + } else { + Test.successful(()) + } _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) case _ if finalStatus == Succeeded => runSuccessfulWorkflowAndVerifyMetadata(workflowDefinition) case _ if finalStatus == Failed => runFailingWorkflowAndVerifyMetadata(workflowDefinition) case _ => Test.invalidTestDefinition("This test can only run successful or failed workflow", workflowDefinition) } - } - def instantAbort(workflowDefinition: Workflow)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { + def instantAbort( + workflowDefinition: Workflow + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = for { _ <- checkDescription(workflowDefinition, validityExpectation = Option(true)) _ <- timingVerificationNotSupported(workflowDefinition.maximumAllowedTime) submittedWorkflow <- submitWorkflow(workflowDefinition) @@ -192,16 +235,21 @@ object TestFormulas extends StrictLogging { workflow = submittedWorkflow, testDefinition = workflowDefinition, expectedStatuses = Set(Running, Aborting, Aborted), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(submittedWorkflow, workflowDefinition, Aborted) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) - def scheduledAbort(workflowDefinition: Workflow, callMarker: CallMarker, restart: Boolean)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def scheduledAbort(workflowDefinition: Workflow, callMarker: CallMarker, restart: Boolean)(implicit + cromwellTracker: Option[CromwellTracker] + ): Test[SubmitResponse] = { def withRestart(): Unit = CentaurConfig.runMode match { case ManagedCromwellServer(_, postRestart, withRestart) if withRestart => CromwellManager.stopCromwell(s"Scheduled restart from ${workflowDefinition.testName}") @@ -217,42 +265,45 @@ object TestFormulas extends StrictLogging { // The Cromwell call status could be running but the backend job might not have started yet, give it some time _ <- waitFor(30.seconds) _ <- abortWorkflow(submittedWorkflow) - _ = if(restart) withRestart() + _ = if (restart) withRestart() _ <- expectSomeProgress( workflow = submittedWorkflow, testDefinition = workflowDefinition, expectedStatuses = Set(Running, Aborting, Aborted), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(submittedWorkflow, workflowDefinition, Aborted) _ <- validatePAPIAborted(workflowDefinition, submittedWorkflow, jobId) // Wait a little to make sure that if the abort didn't work and calls start running we see them in the metadata _ <- waitFor(30.seconds) metadata <- fetchAndValidateNonSubworkflowMetadata(submittedWorkflow, workflowDefinition) - _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(submittedWorkflow, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateDirectoryContentsCounts(workflowDefinition, submittedWorkflow, metadata) } yield SubmitResponse(submittedWorkflow) } def workflowRestart(workflowDefinition: Workflow, - callMarker: CallMarker, - recover: Boolean, - finalStatus: TerminalStatus)( - implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + callMarker: CallMarker, + recover: Boolean, + finalStatus: TerminalStatus + )(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = cromwellRestart(workflowDefinition, callMarker, testRecover = recover, finalStatus = finalStatus) - } - def submitInvalidWorkflow(workflow: Workflow, expectedSubmitResponse: SubmitHttpResponse): Test[SubmitResponse] = { + def submitInvalidWorkflow(workflow: Workflow, expectedSubmitResponse: SubmitHttpResponse): Test[SubmitResponse] = for { _ <- checkDescription(workflow, validityExpectation = None) _ <- timingVerificationNotSupported(workflow.maximumAllowedTime) actualSubmitResponse <- Operations.submitInvalidWorkflow(workflow) _ <- validateSubmitFailure(workflow, expectedSubmitResponse, actualSubmitResponse) } yield actualSubmitResponse - } - def papiUpgrade(workflowDefinition: Workflow, callMarker: CallMarker)(implicit cromwellTracker: Option[CromwellTracker]): Test[SubmitResponse] = { + def papiUpgrade(workflowDefinition: Workflow, callMarker: CallMarker)(implicit + cromwellTracker: Option[CromwellTracker] + ): Test[SubmitResponse] = CentaurConfig.runMode match { case ManagedCromwellServer(_, postRestart, withRestart) if withRestart => for { @@ -266,21 +317,25 @@ object TestFormulas extends StrictLogging { workflow = first, testDefinition = workflowDefinition, expectedStatuses = Set(Running, Succeeded), - timeout = CentaurConfig.workflowProgressTimeout, + timeout = CentaurConfig.workflowProgressTimeout ) _ <- pollUntilStatus(first, workflowDefinition, Succeeded) _ <- checkDescription(workflowDefinition.secondRun, validityExpectation = Option(true)) - second <- runSuccessfulWorkflow(workflowDefinition.secondRun) // Same WDL and config but a "backend" runtime option targeting PAPI v2. + second <- runSuccessfulWorkflow( + workflowDefinition.secondRun + ) // Same WDL and config but a "backend" runtime option targeting PAPI v2. _ <- printHashDifferential(first, second) metadata <- fetchAndValidateNonSubworkflowMetadata(second, workflowDefinition, Option(first.id.id)) - _ <- fetchAndValidateJobManagerStyleMetadata(second, workflowDefinition, prefetchedOriginalNonSubWorkflowMetadata = None) + _ <- fetchAndValidateJobManagerStyleMetadata(second, + workflowDefinition, + prefetchedOriginalNonSubWorkflowMetadata = None + ) _ = cromwellTracker.track(metadata) _ <- validateNoCacheMisses(second, metadata, workflowDefinition) _ <- validateDirectoryContentsCounts(workflowDefinition, second, metadata) } yield SubmitResponse(second) case _ => Test.invalidTestDefinition("Configuration not supported by PapiUpgradeTest", workflowDefinition) } - } implicit class EnhancedCromwellTracker(val tracker: Option[CromwellTracker]) extends AnyVal { def track(metadata: WorkflowMetadata): Unit = tracker foreach { _.track(metadata) } diff --git a/centaur/src/main/scala/centaur/test/markers/CallMarker.scala b/centaur/src/main/scala/centaur/test/markers/CallMarker.scala index 08eafc09213..56274a07ed4 100644 --- a/centaur/src/main/scala/centaur/test/markers/CallMarker.scala +++ b/centaur/src/main/scala/centaur/test/markers/CallMarker.scala @@ -7,12 +7,11 @@ import configs.Result.{Failure, Success} import configs.syntax._ object CallMarker { - def fromConfig(config: Config): ErrorOr[Option[CallMarker]] = { + def fromConfig(config: Config): ErrorOr[Option[CallMarker]] = config.get[Option[String]]("callMark") match { case Success(marker) => (marker map CallMarker.apply).validNel case Failure(f) => s"Invalid restart marker $f".invalidNel } - } } /** diff --git a/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala b/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala index 098328d0d41..f79aaca6826 100644 --- a/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala +++ b/centaur/src/main/scala/centaur/test/metadata/CallAttemptFailure.scala @@ -12,8 +12,7 @@ import io.circe.parser._ * * https://github.com/DataBiosphere/job-manager/blob/f83e4284e2419389b7e515720c9d960d2eb81a29/servers/cromwell/jobs/controllers/jobs_controller.py#L155-L162 */ -case class CallAttemptFailure -( +case class CallAttemptFailure( workflowId: String, callFullyQualifiedName: String, jobIndex: Int, @@ -27,29 +26,25 @@ case class CallAttemptFailure ) object CallAttemptFailure { - def buildFailures(jsonOption: Option[String]): IO[Vector[CallAttemptFailure]] = { + def buildFailures(jsonOption: Option[String]): IO[Vector[CallAttemptFailure]] = jsonOption.map(buildFailures).getOrElse(IO.pure(Vector.empty)) - } - def buildFailures(json: String): IO[Vector[CallAttemptFailure]] = { + def buildFailures(json: String): IO[Vector[CallAttemptFailure]] = IO.fromEither(decode[Vector[CallAttemptFailure]](json)) - } - private implicit val decodeFailures: Decoder[Vector[CallAttemptFailure]] = { + implicit private val decodeFailures: Decoder[Vector[CallAttemptFailure]] = Decoder.instance { c => for { workflowId <- c.get[String]("id") calls <- c.get[Map[String, Json]]("calls").map(_.toVector) - callAttemptFailures <- calls.flatTraverse[Decoder.Result, CallAttemptFailure] { - case (callName, callJson) => - val decoderCallAttempt = decodeFromCallAttempt(workflowId, callName) - callJson.as[Vector[Option[CallAttemptFailure]]](Decoder.decodeVector(decoderCallAttempt)).map(_.flatten) + callAttemptFailures <- calls.flatTraverse[Decoder.Result, CallAttemptFailure] { case (callName, callJson) => + val decoderCallAttempt = decodeFromCallAttempt(workflowId, callName) + callJson.as[Vector[Option[CallAttemptFailure]]](Decoder.decodeVector(decoderCallAttempt)).map(_.flatten) } } yield callAttemptFailures } or Decoder.const(Vector.empty) - } - private def decodeFromCallAttempt(workflowId: String, callName: String): Decoder[Option[CallAttemptFailure]] = { + private def decodeFromCallAttempt(workflowId: String, callName: String): Decoder[Option[CallAttemptFailure]] = Decoder.instance { c => for { shardIndexOption <- c.get[Option[Int]]("shardIndex") @@ -77,5 +72,4 @@ object CallAttemptFailure { } } yield callAttemptFailureOption } or Decoder.const(None) - } } diff --git a/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala b/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala index 182d1fad58f..5054105c154 100644 --- a/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala +++ b/centaur/src/main/scala/centaur/test/metadata/WorkflowFlatMetadata.scala @@ -17,7 +17,6 @@ import spray.json._ import scala.language.postfixOps import scala.util.{Failure, Success, Try} - /** * Workflow metadata that has been flattened for Centaur test purposes. The keys are similar to the simpleton-syntax * stored in the Cromwell database, and values are primitive types, not nested JSON objects or arrays. @@ -27,15 +26,23 @@ case class WorkflowFlatMetadata(value: Map[String, JsValue]) extends AnyVal { def diff(actual: WorkflowFlatMetadata, workflowID: UUID, cacheHitUUID: Option[UUID] = None): Iterable[String] = { // If the test fails in initialization there wouldn't be workflow root metadata, and if that's the expectation // then that's ok. - val workflowRoot = actual.value.get("workflowRoot").collectFirst { case JsString(r) => r } getOrElse "No Workflow Root" + val workflowRoot = + actual.value.get("workflowRoot").collectFirst { case JsString(r) => r } getOrElse "No Workflow Root" val missingErrors = value.keySet.diff(actual.value.keySet) map { k => s"Missing key: $k" } - val mismatchErrors = value.keySet.intersect(actual.value.keySet) flatMap { k => diffValues(k, value(k), actual.value(k), - workflowID, workflowRoot, cacheHitUUID)} + val mismatchErrors = value.keySet.intersect(actual.value.keySet) flatMap { k => + diffValues(k, value(k), actual.value(k), workflowID, workflowRoot, cacheHitUUID) + } mismatchErrors ++ missingErrors } - private def diffValues(key: String, expected: JsValue, actual: JsValue, workflowID: UUID, workflowRoot: String, cacheHitUUID: Option[UUID]): Option[String] = { + private def diffValues(key: String, + expected: JsValue, + actual: JsValue, + workflowID: UUID, + workflowRoot: String, + cacheHitUUID: Option[UUID] + ): Option[String] = { /* FIXME/TODO: @@ -59,8 +66,10 @@ case class WorkflowFlatMetadata(value: Map[String, JsValue]) extends AnyVal { val stripped = stripQuotes(cacheSubstitutions).stripPrefix("~~") (!stripQuotes(o.toString).contains(stripped)).option(s"Actual value ${o.toString()} does not contain $stripped") case o: JsString => (cacheSubstitutions != o.toString).option(s"expected: $cacheSubstitutions but got: $actual") - case o: JsNumber => (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") - case o: JsBoolean => (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") + case o: JsNumber => + (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") + case o: JsBoolean => + (expected != JsString(o.value.toString)).option(s"expected: $cacheSubstitutions but got: $actual") case o: JsArray if stripQuotes(cacheSubstitutions).startsWith("~>") => val stripped = stripQuotes(cacheSubstitutions).stripPrefix("~>") val replaced = stripped.replaceAll("\\\\\"", "\"") @@ -76,12 +85,11 @@ case class WorkflowFlatMetadata(value: Map[String, JsValue]) extends AnyVal { object WorkflowFlatMetadata { - def fromConfig(config: Config): ErrorOr[WorkflowFlatMetadata] = { + def fromConfig(config: Config): ErrorOr[WorkflowFlatMetadata] = config.extract[Map[String, Option[String]]] match { case Result.Success(m) => Valid(WorkflowFlatMetadata(m safeMapValues { _.map(JsString.apply).getOrElse(JsNull) })) case Result.Failure(_) => invalidNel(s"Metadata block can not be converted to a Map: $config") } - } def fromWorkflowMetadata(workflowMetadata: WorkflowMetadata): ErrorOr[WorkflowFlatMetadata] = { val jsValue: ErrorOr[JsValue] = Try(workflowMetadata.value.parseJson) match { @@ -96,9 +104,8 @@ object WorkflowFlatMetadata { } implicit class EnhancedWorkflowMetadata(val workflowMetadata: WorkflowMetadata) { - def asFlat: WorkflowFlatMetadata = { + def asFlat: WorkflowFlatMetadata = WorkflowFlatMetadata.fromWorkflowMetadata(workflowMetadata).unsafe - } } implicit class EnhancedWorkflowFlatMetadata(val workflowFlatMetadata: WorkflowFlatMetadata) { @@ -113,25 +120,22 @@ object WorkflowFlatMetadata { } implicit class EnhancedExpectation(val expectation: String) extends AnyVal { - def replaceExpectationVariables(workflowId: WorkflowId, workflowRoot: String): String = { + def replaceExpectationVariables(workflowId: WorkflowId, workflowRoot: String): String = expectation.replaceAll("<>", workflowId.toString).replaceAll("<>", workflowRoot) - } } } object WorkflowFlatOutputs { implicit class EnhancedWorkflowOutputs(val workflowOutputs: WorkflowOutputs) extends AnyVal { - def asFlat: WorkflowFlatMetadata = { + def asFlat: WorkflowFlatMetadata = workflowOutputs.outputs.asMap map WorkflowFlatMetadata.apply unsafe - } } } object WorkflowFlatLabels { implicit class EnhancedWorkflowLabels(val workflowLabels: WorkflowLabels) extends AnyVal { - def asFlat: WorkflowFlatMetadata = { + def asFlat: WorkflowFlatMetadata = workflowLabels.labels.asMap map WorkflowFlatMetadata.apply unsafe - } } } @@ -140,11 +144,10 @@ object JsValueEnhancer { import DefaultJsonProtocol._ import centaur.json.JsonUtils._ - def asMap: ErrorOr[Map[String, JsValue]] = { + def asMap: ErrorOr[Map[String, JsValue]] = Try(jsValue.asJsObject.flatten().convertTo[Map[String, JsValue]]) match { case Success(m) => Valid(m) case Failure(e) => invalidNel(s"Unable to convert JsValue to JsObject: ${e.getMessage}") } - } } } diff --git a/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala b/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala index 258c211c900..e462ea83c90 100644 --- a/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala +++ b/centaur/src/main/scala/centaur/test/standard/CentaurTestCase.scala @@ -19,8 +19,8 @@ case class CentaurTestCase(workflow: Workflow, testFormat: CentaurTestFormat, testOptions: TestOptions, submittedWorkflowTracker: SubmittedWorkflowTracker, - submitResponseOption: Option[SubmitHttpResponse])( - implicit cromwellTracker: Option[CromwellTracker]) { + submitResponseOption: Option[SubmitHttpResponse] +)(implicit cromwellTracker: Option[CromwellTracker]) { def testFunction: Test[SubmitResponse] = this.testFormat match { case WorkflowSuccessTest => TestFormulas.runSuccessfulWorkflowAndVerifyMetadata(workflow) @@ -32,10 +32,14 @@ case class CentaurTestCase(workflow: Workflow, case RunFailingTwiceExpectingNoCallCachingTest => TestFormulas.runFailingWorkflowTwiceExpectingNoCaching(workflow) case SubmitFailureTest => TestFormulas.submitInvalidWorkflow(workflow, submitResponseOption.get) case InstantAbort => TestFormulas.instantAbort(workflow) - case CromwellRestartWithRecover(callMarker)=> TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Succeeded) - case WorkflowFailureRestartWithRecover(callMarker)=> TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Failed) - case WorkflowFailureRestartWithoutRecover(callMarker)=> TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Failed) - case CromwellRestartWithoutRecover(callMarker) => TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Succeeded) + case CromwellRestartWithRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Succeeded) + case WorkflowFailureRestartWithRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = true, finalStatus = Failed) + case WorkflowFailureRestartWithoutRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Failed) + case CromwellRestartWithoutRecover(callMarker) => + TestFormulas.workflowRestart(workflow, callMarker, recover = false, finalStatus = Succeeded) case ScheduledAbort(callMarker) => TestFormulas.scheduledAbort(workflow, callMarker, restart = false) case ScheduledAbortWithRestart(callMarker) => TestFormulas.scheduledAbort(workflow, callMarker, restart = true) case PapiUpgradeTest(callMarker) => TestFormulas.papiUpgrade(workflow, callMarker) @@ -44,9 +48,15 @@ case class CentaurTestCase(workflow: Workflow, def isIgnored(supportedBackends: List[String]): Boolean = { val backendSupported = workflow.backends match { - case AllBackendsRequired(allBackends) => allBackends forall supportedBackends.contains - case AnyBackendRequired(anyBackend) => anyBackend exists supportedBackends.contains - case OnlyBackendsAllowed(onlyBackends) => supportedBackends forall onlyBackends.contains + case AllBackendsRequired(testBackends) => + // Test will run on servers that support all of the test's backends (or more) (default) + testBackends forall supportedBackends.contains + case AnyBackendRequired(testBackends) => + // Test will run on servers that support at least one of the test's backends (or more) + testBackends exists supportedBackends.contains + case OnlyBackendsAllowed(testBackends) => + // Test will run on servers that only support backends the test specifies (or fewer) + supportedBackends forall testBackends.contains } testOptions.ignore || !backendSupported @@ -58,13 +68,14 @@ case class CentaurTestCase(workflow: Workflow, } object CentaurTestCase { - def fromFile(cromwellTracker: Option[CromwellTracker])(file: File): ErrorOr[CentaurTestCase] = { + def fromFile(cromwellTracker: Option[CromwellTracker])(file: File): ErrorOr[CentaurTestCase] = Try(ConfigFactory.parseFile(file.toJava).resolve()) match { case Success(c) => - CentaurTestCase.fromConfig(c, file.parent, cromwellTracker) flatMap validateTestCase leftMap { s"Error in test file '$file'." :: _ } + CentaurTestCase.fromConfig(c, file.parent, cromwellTracker) flatMap validateTestCase leftMap { + s"Error in test file '$file'." :: _ + } case Failure(f) => invalidNel(s"Invalid test config: $file (${f.getMessage})") } - } def fromConfig(conf: Config, configFile: File, cromwellTracker: Option[CromwellTracker]): ErrorOr[CentaurTestCase] = { val submittedWorkflowTracker = new SubmittedWorkflowTracker() @@ -77,18 +88,18 @@ object CentaurTestCase { } } - private def validateTestCase(testCase: CentaurTestCase): ErrorOr[CentaurTestCase] = { + private def validateTestCase(testCase: CentaurTestCase): ErrorOr[CentaurTestCase] = testCase.testFormat match { - case SubmitFailureTest => validateSubmitFailure(testCase.workflow, testCase.submitResponseOption).map(_ => testCase) + case SubmitFailureTest => + validateSubmitFailure(testCase.workflow, testCase.submitResponseOption).map(_ => testCase) case _ => Valid(testCase) } - } private def validateSubmitFailure(workflow: Workflow, - submitResponseOption: Option[SubmitHttpResponse]): ErrorOr[SubmitResponse] = { + submitResponseOption: Option[SubmitHttpResponse] + ): ErrorOr[SubmitResponse] = submitResponseOption match { case None => invalidNel("No submit stanza included in test config") case Some(response) => Valid(response) } - } } diff --git a/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala b/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala index 1cb2345f25c..eae961d6e5f 100644 --- a/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala +++ b/centaur/src/main/scala/centaur/test/standard/CentaurTestFormat.scala @@ -9,7 +9,7 @@ import configs.syntax._ sealed abstract class CentaurTestFormat(val name: String) { val lowerCaseName = name.toLowerCase - + def testSpecString: String = this match { case WorkflowSuccessTest => "successfully run" case WorkflowSuccessAndTimedOutputsTest => "successfully run" @@ -20,12 +20,14 @@ sealed abstract class CentaurTestFormat(val name: String) { case RunFailingTwiceExpectingNoCallCachingTest => "Fail the first run and NOT call cache the second run of" case SubmitFailureTest => "fail to submit" case InstantAbort => "abort a workflow immediately after submission" - case _: PapiUpgradeTest => "make sure a PAPI upgrade preserves call caching when the `name-for-call-caching-purposes` attribute is used" + case _: PapiUpgradeTest => + "make sure a PAPI upgrade preserves call caching when the `name-for-call-caching-purposes` attribute is used" case _: CromwellRestartWithRecover => "survive a Cromwell restart and recover jobs" case _: CromwellRestartWithoutRecover => "survive a Cromwell restart" case _: ScheduledAbort => "abort a workflow mid run" case _: ScheduledAbortWithRestart => "abort a workflow mid run and restart immediately" - case _: WorkflowFailureRestartWithRecover => "survive a Cromwell restart when a workflow was failing and recover jobs" + case _: WorkflowFailureRestartWithRecover => + "survive a Cromwell restart when a workflow was failing and recover jobs" case _: WorkflowFailureRestartWithoutRecover => "survive a Cromwell restart when a workflow was failing" case other => s"unrecognized format $other" } @@ -39,71 +41,89 @@ object CentaurTestFormat { sealed trait SequentialTestFormat extends CentaurTestFormat { override def isParallel: Boolean = false } - + sealed trait RestartFormat extends SequentialTestFormat - sealed trait WithCallMarker { this: CentaurTestFormat => val build: CallMarker => CentaurTestFormat } - + sealed trait WithCallMarker { this: CentaurTestFormat => + val build: CallMarker => CentaurTestFormat + } + case object WorkflowSuccessTest extends CentaurTestFormat("WorkflowSuccess") case object WorkflowSuccessAndTimedOutputsTest extends CentaurTestFormat("WorkflowSuccessAndTimedOutputs") case object WorkflowFailureTest extends CentaurTestFormat("WorkflowFailure") case object RunTwiceExpectingCallCachingTest extends CentaurTestFormat("RunTwiceExpectingCallCaching") case object RunThriceExpectingCallCachingTest extends CentaurTestFormat(name = "RunThriceExpectingCallCaching") case object RunTwiceExpectingNoCallCachingTest extends CentaurTestFormat("RunTwiceExpectingNoCallCaching") - case object RunFailingTwiceExpectingNoCallCachingTest extends CentaurTestFormat("RunFailingTwiceExpectingNoCallCaching") + case object RunFailingTwiceExpectingNoCallCachingTest + extends CentaurTestFormat("RunFailingTwiceExpectingNoCallCaching") case object SubmitFailureTest extends CentaurTestFormat("SubmitFailure") case object InstantAbort extends CentaurTestFormat("InstantAbort") with SequentialTestFormat object CromwellRestartWithRecover extends CentaurTestFormat("CromwellRestartWithRecover") with WithCallMarker { val build = CromwellRestartWithRecover.apply _ } - case class CromwellRestartWithRecover(callMarker: CallMarker) extends CentaurTestFormat(CromwellRestartWithRecover.name) with RestartFormat - + case class CromwellRestartWithRecover(callMarker: CallMarker) + extends CentaurTestFormat(CromwellRestartWithRecover.name) + with RestartFormat + object CromwellRestartWithoutRecover extends CentaurTestFormat("CromwellRestartWithoutRecover") with WithCallMarker { val build = CromwellRestartWithoutRecover.apply _ } - case class CromwellRestartWithoutRecover(callMarker: CallMarker) extends CentaurTestFormat(CromwellRestartWithoutRecover.name) with RestartFormat + case class CromwellRestartWithoutRecover(callMarker: CallMarker) + extends CentaurTestFormat(CromwellRestartWithoutRecover.name) + with RestartFormat object ScheduledAbort extends CentaurTestFormat("ScheduledAbort") with WithCallMarker { val build = ScheduledAbort.apply _ } - case class ScheduledAbort(callMarker: CallMarker) extends CentaurTestFormat(ScheduledAbort.name) with SequentialTestFormat + case class ScheduledAbort(callMarker: CallMarker) + extends CentaurTestFormat(ScheduledAbort.name) + with SequentialTestFormat object ScheduledAbortWithRestart extends CentaurTestFormat("ScheduledAbortWithRestart") with WithCallMarker { val build = ScheduledAbortWithRestart.apply _ } - case class ScheduledAbortWithRestart(callMarker: CallMarker) extends CentaurTestFormat(ScheduledAbortWithRestart.name) with RestartFormat + case class ScheduledAbortWithRestart(callMarker: CallMarker) + extends CentaurTestFormat(ScheduledAbortWithRestart.name) + with RestartFormat - object WorkflowFailureRestartWithRecover extends CentaurTestFormat("WorkflowFailureRestartWithRecover") with WithCallMarker { + object WorkflowFailureRestartWithRecover + extends CentaurTestFormat("WorkflowFailureRestartWithRecover") + with WithCallMarker { val build = WorkflowFailureRestartWithRecover.apply _ } - case class WorkflowFailureRestartWithRecover(callMarker: CallMarker) extends CentaurTestFormat(WorkflowFailureRestartWithRecover.name) with RestartFormat + case class WorkflowFailureRestartWithRecover(callMarker: CallMarker) + extends CentaurTestFormat(WorkflowFailureRestartWithRecover.name) + with RestartFormat - object WorkflowFailureRestartWithoutRecover extends CentaurTestFormat("WorkflowFailureRestartWithoutRecover") with WithCallMarker { + object WorkflowFailureRestartWithoutRecover + extends CentaurTestFormat("WorkflowFailureRestartWithoutRecover") + with WithCallMarker { val build = WorkflowFailureRestartWithoutRecover.apply _ } - case class WorkflowFailureRestartWithoutRecover(callMarker: CallMarker) extends CentaurTestFormat(WorkflowFailureRestartWithoutRecover.name) with RestartFormat + case class WorkflowFailureRestartWithoutRecover(callMarker: CallMarker) + extends CentaurTestFormat(WorkflowFailureRestartWithoutRecover.name) + with RestartFormat object PapiUpgradeTest extends CentaurTestFormat("PapiUpgrade") with WithCallMarker { val build = PapiUpgradeTest.apply _ } case class PapiUpgradeTest(callMarker: CallMarker) extends CentaurTestFormat(PapiUpgradeTest.name) with RestartFormat - def fromConfig(conf: Config): Checked[CentaurTestFormat] = { - + def fromConfig(conf: Config): Checked[CentaurTestFormat] = CallMarker.fromConfig(conf).toEither flatMap { callMarker => conf.get[String]("testFormat") match { case Success(f) => CentaurTestFormat.fromString(f, callMarker) case Failure(_) => "No testFormat string provided".invalidNelCheck[CentaurTestFormat] } } - } private def fromString(testFormat: String, callMarker: Option[CallMarker]): Checked[CentaurTestFormat] = { def withCallMarker(name: String, constructor: CallMarker => CentaurTestFormat) = callMarker match { case Some(marker) => constructor(marker).validNelCheck - case None => s"$name needs a callMarker to know on which call to trigger the restart".invalidNelCheck[CentaurTestFormat] + case None => + s"$name needs a callMarker to know on which call to trigger the restart".invalidNelCheck[CentaurTestFormat] } - + List( WorkflowSuccessTest, WorkflowSuccessAndTimedOutputsTest, @@ -121,9 +141,10 @@ object CentaurTestFormat { WorkflowFailureRestartWithRecover, WorkflowFailureRestartWithoutRecover, PapiUpgradeTest - ).collectFirst({ - case format: WithCallMarker if format.name.equalsIgnoreCase(testFormat) => withCallMarker(format.name, format.build) + ).collectFirst { + case format: WithCallMarker if format.name.equalsIgnoreCase(testFormat) => + withCallMarker(format.name, format.build) case format if format.name.equalsIgnoreCase(testFormat) => format.validNelCheck - }).getOrElse(s"No such test format: $testFormat".invalidNelCheck[CentaurTestFormat]) + }.getOrElse(s"No such test format: $testFormat".invalidNelCheck[CentaurTestFormat]) } } diff --git a/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala b/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala index ed160482bef..faed88cc8cd 100644 --- a/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala +++ b/centaur/src/main/scala/centaur/test/submit/SubmitResponse.scala @@ -14,13 +14,11 @@ import cromwell.api.model.SubmittedWorkflow sealed trait SubmitResponse object SubmitResponse { - def apply(submittedWorkflow: SubmittedWorkflow): SubmitResponse = { + def apply(submittedWorkflow: SubmittedWorkflow): SubmitResponse = SubmitWorkflowResponse(submittedWorkflow) - } - def apply(statusCode: Int, message: String): SubmitResponse = { + def apply(statusCode: Int, message: String): SubmitResponse = SubmitHttpResponse(statusCode, message) - } } case class SubmitWorkflowResponse(submittedWorkflow: SubmittedWorkflow) extends SubmitResponse @@ -29,7 +27,7 @@ case class SubmitHttpResponse(statusCode: Int, message: String) extends SubmitRe object SubmitHttpResponse { - def fromConfig(conf: Config): ErrorOr[Option[SubmitHttpResponse]] = { + def fromConfig(conf: Config): ErrorOr[Option[SubmitHttpResponse]] = conf.get[Config]("submit") match { case Result.Failure(_) => Valid(None) case Result.Success(submitConf) => @@ -41,17 +39,13 @@ object SubmitHttpResponse { Option(_) } } - } - private def toErrorOr[A](result: Result[A]): ErrorOr[A] = { + private def toErrorOr[A](result: Result[A]): ErrorOr[A] = result match { case Result.Success(value) => Valid(value) case Result.Failure(error) => - error.messages - .toList - .toNel + error.messages.toList.toNel .getOrElse(throw new RuntimeException("Paranoia... error.messages is a Nel exposed as a Seq.")) .invalid } - } } diff --git a/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala b/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala index c5813165aea..4d064455f8c 100644 --- a/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala +++ b/centaur/src/main/scala/centaur/test/workflow/DirectoryContentCountCheck.scala @@ -2,7 +2,7 @@ package centaur.test.workflow import cats.data.Validated._ import cats.syntax.all._ -import centaur.test.{AWSFilesChecker, FilesChecker, LocalFilesChecker, PipelinesFilesChecker} +import centaur.test.{AWSFilesChecker, BlobFilesChecker, FilesChecker, LocalFilesChecker, PipelinesFilesChecker} import com.typesafe.config.Config import common.validation.ErrorOr.ErrorOr import configs.Result @@ -16,17 +16,23 @@ object DirectoryContentCountCheck { if (!keepGoing) { valid(None) } else { - val directoryContentCountsValidation: ErrorOr[Map[String, Int]] = conf.get[Map[String, Int]]("outputExpectations") match { - case Result.Success(a) => valid(a) - case Result.Failure(_) => invalidNel(s"Test '$name': Unable to read outputExpectations as a Map[String, Int]") - } + val directoryContentCountsValidation: ErrorOr[Map[String, Int]] = + conf.get[Map[String, Int]]("outputExpectations") match { + case Result.Success(a) => valid(a) + case Result.Failure(_) => invalidNel(s"Test '$name': Unable to read outputExpectations as a Map[String, Int]") + } val fileSystemChecker: ErrorOr[FilesChecker] = conf.get[String]("fileSystemCheck") match { case Result.Success("gcs") => valid(PipelinesFilesChecker) case Result.Success("local") => valid(LocalFilesChecker) case Result.Success("aws") => valid(AWSFilesChecker) - case Result.Success(_) => invalidNel(s"Test '$name': Invalid 'fileSystemCheck' value (must be either 'local', 'gcs' or 'aws'") - case Result.Failure(_) => invalidNel(s"Test '$name': Must specify a 'fileSystemCheck' value (must be either 'local', 'gcs' or 'aws'") + case Result.Success("blob") => valid(BlobFilesChecker) + case Result.Success(_) => + invalidNel(s"Test '$name': Invalid 'fileSystemCheck' value (must be either 'local', 'gcs', 'blob', or 'aws'") + case Result.Failure(_) => + invalidNel( + s"Test '$name': Must specify a 'fileSystemCheck' value (must be either 'local', 'gcs', 'blob', or 'aws'" + ) } (directoryContentCountsValidation, fileSystemChecker) mapN { (d, f) => Option(DirectoryContentCountCheck(d, f)) } diff --git a/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala b/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala index dc3f0355f80..718db740dab 100644 --- a/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala +++ b/centaur/src/main/scala/centaur/test/workflow/SubmittedWorkflowTracker.scala @@ -25,7 +25,6 @@ class SubmittedWorkflowTracker { * object require a retry. Prevents unwanted cache hits from partially successful attempts when retrying a call * caching test case. */ - def add(submittedWorkflow: SubmittedWorkflow): Unit = { + def add(submittedWorkflow: SubmittedWorkflow): Unit = submittedWorkflowIds = submittedWorkflow.id :: submittedWorkflowIds - } } diff --git a/centaur/src/main/scala/centaur/test/workflow/Workflow.scala b/centaur/src/main/scala/centaur/test/workflow/Workflow.scala index 743924bb9c6..212064acfe1 100644 --- a/centaur/src/main/scala/centaur/test/workflow/Workflow.scala +++ b/centaur/src/main/scala/centaur/test/workflow/Workflow.scala @@ -15,17 +15,18 @@ import cromwell.api.model.{WorkflowDescribeRequest, WorkflowSingleSubmission} import java.nio.file.Path import scala.concurrent.duration.FiniteDuration -final case class Workflow private(testName: String, - data: WorkflowData, - metadata: Option[WorkflowFlatMetadata], - notInMetadata: List[String], - directoryContentCounts: Option[DirectoryContentCountCheck], - backends: BackendsRequirement, - retryTestFailures: Boolean, - allowOtherOutputs: Boolean, - skipDescribeEndpointValidation: Boolean, - submittedWorkflowTracker: SubmittedWorkflowTracker, - maximumAllowedTime: Option[FiniteDuration]) { +final case class Workflow private (testName: String, + data: WorkflowData, + metadata: Option[WorkflowFlatMetadata], + notInMetadata: List[String], + directoryContentCounts: Option[DirectoryContentCountCheck], + backends: BackendsRequirement, + retryTestFailures: Boolean, + allowOtherOutputs: Boolean, + skipDescribeEndpointValidation: Boolean, + submittedWorkflowTracker: SubmittedWorkflowTracker, + maximumAllowedTime: Option[FiniteDuration] +) { def toWorkflowSubmission: WorkflowSingleSubmission = WorkflowSingleSubmission( workflowSource = data.workflowContent, @@ -36,7 +37,8 @@ final case class Workflow private(testName: String, inputsJson = data.inputs.map(_.unsafeRunSync()), options = data.options.map(_.unsafeRunSync()), labels = Option(data.labels), - zippedImports = data.zippedImports) + zippedImports = data.zippedImports + ) def toWorkflowDescribeRequest: WorkflowDescribeRequest = WorkflowDescribeRequest( workflowSource = data.workflowContent, @@ -46,22 +48,26 @@ final case class Workflow private(testName: String, inputsJson = data.inputs.map(_.unsafeRunSync()) ) - def secondRun: Workflow = { + def secondRun: Workflow = copy(data = data.copy(options = data.secondOptions)) - } - def thirdRun: Workflow = { + def thirdRun: Workflow = copy(data = data.copy(options = data.thirdOptions)) - } } object Workflow { - def fromConfig(conf: Config, configFile: File, submittedWorkflowTracker: SubmittedWorkflowTracker): ErrorOr[Workflow] = { + def fromConfig(conf: Config, + configFile: File, + submittedWorkflowTracker: SubmittedWorkflowTracker + ): ErrorOr[Workflow] = conf.get[String]("name") match { case Result.Success(n) => // If backend is provided, Centaur will only run this test if that backend is available on Cromwell - val backendsRequirement = BackendsRequirement.fromConfig(conf.get[String]("backendsMode").map(_.toLowerCase).valueOrElse("all"), conf.get[List[String]]("backends").valueOrElse(List.empty[String]).map(_.toLowerCase)) + val backendsRequirement = BackendsRequirement.fromConfig( + conf.get[String]("backendsMode").map(_.toLowerCase).valueOrElse("all"), + conf.get[List[String]]("backends").valueOrElse(List.empty[String]).map(_.toLowerCase) + ) // If basePath is provided it'll be used as basis for finding other files, otherwise use the dir the config was in val basePath = conf.get[Option[Path]]("basePath") valueOrElse None map (File(_)) getOrElse configFile val metadata: ErrorOr[Option[WorkflowFlatMetadata]] = conf.get[Config]("metadata") match { @@ -73,7 +79,8 @@ object Workflow { case Result.Failure(_) => List.empty } - val directoryContentCheckValidation: ErrorOr[Option[DirectoryContentCountCheck]] = DirectoryContentCountCheck.forConfig(n, conf) + val directoryContentCheckValidation: ErrorOr[Option[DirectoryContentCountCheck]] = + DirectoryContentCountCheck.forConfig(n, conf) val files = conf.get[Config]("files") match { case Result.Success(f) => WorkflowData.fromConfig(filesConfig = f, fullConfig = conf, basePath = basePath) case Result.Failure(_) => invalidNel(s"No 'files' block in $configFile") @@ -89,10 +96,21 @@ object Workflow { val maximumTime: Option[FiniteDuration] = conf.get[Option[FiniteDuration]]("maximumTime").value (files, directoryContentCheckValidation, metadata, retryTestFailuresErrorOr) mapN { - (f, d, m, retryTestFailures) => Workflow(n, f, m, absentMetadata, d, backendsRequirement, retryTestFailures, allowOtherOutputs, validateDescription, submittedWorkflowTracker, maximumTime) + (f, d, m, retryTestFailures) => + Workflow(n, + f, + m, + absentMetadata, + d, + backendsRequirement, + retryTestFailures, + allowOtherOutputs, + validateDescription, + submittedWorkflowTracker, + maximumTime + ) } case Result.Failure(_) => invalidNel(s"No test 'name' for: $configFile") } - } } diff --git a/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala b/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala index 72e66b31e18..2096a5f88d7 100644 --- a/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala +++ b/centaur/src/main/scala/centaur/test/workflow/WorkflowData.scala @@ -34,7 +34,8 @@ case class WorkflowData(workflowContent: Option[String], labels: List[Label], zippedImports: Option[File], secondOptions: Option[IO[String]] = None, - thirdOptions: Option[IO[String]] = None) + thirdOptions: Option[IO[String]] = None +) object WorkflowData { val blockingEC = ExecutionContext.fromExecutorService(Executors.newFixedThreadPool(5)) @@ -47,55 +48,64 @@ object WorkflowData { val workflowSourcePath = filesConfig.as[Option[String]]("workflow") (workflowSourcePath, workflowUrl) match { - case (Some(workflowPath), None) => Valid(WorkflowData( - workflowPath = Option(workflowPath), - workflowUrl = None, - filesConfig = filesConfig, - fullConfig = fullConfig, - basePath = basePath)) - case (None, Some(_)) => Valid(WorkflowData( - workflowPath = None, - workflowUrl = workflowUrl, - filesConfig = filesConfig, - fullConfig = fullConfig, - basePath = basePath)) + case (Some(workflowPath), None) => + Valid( + WorkflowData(workflowPath = Option(workflowPath), + workflowUrl = None, + filesConfig = filesConfig, + fullConfig = fullConfig, + basePath = basePath + ) + ) + case (None, Some(_)) => + Valid( + WorkflowData(workflowPath = None, + workflowUrl = workflowUrl, + filesConfig = filesConfig, + fullConfig = fullConfig, + basePath = basePath + ) + ) case (Some(_), Some(_)) => invalidNel(s"Both 'workflow' path or 'workflowUrl' can't be provided.") case (None, None) => invalidNel(s"No 'workflow' path or 'workflowUrl' provided.") } } - def apply(workflowPath: Option[String], workflowUrl: Option[String], filesConfig: Config, fullConfig: Config, basePath: File): WorkflowData = { + def apply(workflowPath: Option[String], + workflowUrl: Option[String], + filesConfig: Config, + fullConfig: Config, + basePath: File + ): WorkflowData = { def slurp(file: String): IO[String] = file match { - case http if http.startsWith("http://") || http.startsWith("https://") => + case http if http.startsWith("http://") || http.startsWith("https://") => httpClient.expect[String](http) case gcs if gcs.startsWith("gs://") => val noScheme = gcs.stripPrefix("gs://") val firstSlashPosition = noScheme.indexOf("/") val blob = BlobId.of(noScheme.substring(0, firstSlashPosition), noScheme.substring(firstSlashPosition + 1)) - IO { gcsStorage.readAllBytes(blob).map(_.toChar).mkString } + IO(gcsStorage.readAllBytes(blob).map(_.toChar).mkString) case local => - IO { basePath./(local).contentAsString } + IO(basePath./(local).contentAsString) } - - def getOptionalFileContent(name: String): Option[IO[String]] = { + + def getOptionalFileContent(name: String): Option[IO[String]] = filesConfig.getAs[String](name).map(slurp) - } def getImports = filesConfig.get[List[String]]("imports") match { case Success(paths) => zipImports(paths map basePath./) case Failure(_) => None } - def getImportsDirName(workflowPath: Option[File], workflowUrl: Option[String]): String = { + def getImportsDirName(workflowPath: Option[File], workflowUrl: Option[String]): String = workflowPath match { case Some(file) => file.name.replaceAll("\\.[^.]*$", "") case None => // workflow url is defined val fileName = workflowUrl.get.split("/").last fileName.replaceAll("\\.[^.]*$", "") } - } - def zipImports(imports: List[File]): Option[File] = { + def zipImports(imports: List[File]): Option[File] = imports match { case Nil => None case _ => @@ -109,7 +119,6 @@ object WorkflowData { Option(importsDir.zip()) } - } def getLabels: List[Label] = { import cromwell.api.model.LabelsJsonFormatter._ diff --git a/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala b/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala index afd3918d738..7069cee9c40 100644 --- a/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala +++ b/centaur/src/test/scala/centaur/api/DaemonizedDefaultThreadFactorySpec.scala @@ -9,7 +9,7 @@ class DaemonizedDefaultThreadFactorySpec extends AnyFlatSpec with CromwellTimeou behavior of "DaemonizedDefaultThreadFactory" it should "create a non-blocking execution context" in { - val thread = DaemonizedDefaultThreadFactory.newThread(() => {}) + val thread = DaemonizedDefaultThreadFactory.newThread { () => } thread.getName should startWith("daemonpool-thread-") thread.isDaemon should be(true) } diff --git a/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala b/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala index 1f89477d388..91fe7123ed8 100644 --- a/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala +++ b/centaur/src/test/scala/centaur/json/JsonUtilsSpec.scala @@ -111,7 +111,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "calls.wf_hello.hello.outputs.salutation" -> "Hello Mr. Bean!", "calls.wf_hello.hello.runtimeAttributes.bootDiskSizeGb" -> "10", "calls.wf_hello.hello.runtimeAttributes.continueOnReturnCode" -> "0", - "calls.wf_hello.hello.runtimeAttributes.maxRetries" -> "0", + "calls.wf_hello.hello.runtimeAttributes.maxRetries" -> "0" ).map(x => (x._1, JsString(x._2))) val actualFlattenedMetadata: Map[String, JsValue] = metadata.parseJson.asJsObject.flatten().fields @@ -169,7 +169,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "calls.wf_hello.task1.executionEvents.0.description" -> "task 1 step 1", "calls.wf_hello.task2.executionEvents.0.description" -> "task 2 step 1", "calls.wf_hello.task1.runtimeAttributes.bootDiskSizeGb" -> "10", - "calls.wf_hello.task2.runtimeAttributes.bootDiskSizeGb" -> "10", + "calls.wf_hello.task2.runtimeAttributes.bootDiskSizeGb" -> "10" ).map(x => (x._1, JsString(x._2))) val actualFlattenedMetadata: Map[String, JsValue] = metadata.parseJson.asJsObject.flatten().fields @@ -249,7 +249,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "wf_hello.hello.0.2.shardIndex" -> 0, "wf_hello.hello.1.shardIndex" -> 1, "wf_hello.hello.1.1.shardIndex" -> 1, - "wf_hello.hello.1.2.shardIndex" -> 1, + "wf_hello.hello.1.2.shardIndex" -> 1 ).map(x => (x._1, JsNumber(x._2))) ++ Map( "id" -> "5abfaa90-570f-48d4-a35b-81d5ad4ea0f7", "status" -> "Succeeded", @@ -265,7 +265,7 @@ class JsonUtilsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "wf_hello.hello.0.2.runtimeAttributes.memory" -> "1.1 GB", "wf_hello.hello.1.runtimeAttributes.memory" -> "1.1 GB", "wf_hello.hello.1.1.runtimeAttributes.memory" -> "1 GB", - "wf_hello.hello.1.2.runtimeAttributes.memory" -> "1.1 GB", + "wf_hello.hello.1.2.runtimeAttributes.memory" -> "1.1 GB" ).map(x => (x._1, JsString(x._2))) val actualFlattenedMetadata: Map[String, JsValue] = metadata.parseJson.asJsObject.flatten().fields diff --git a/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala b/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala index 9d09827027a..28a74274192 100644 --- a/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala +++ b/centaur/src/test/scala/centaur/test/CentaurOperationsSpec.scala @@ -15,23 +15,29 @@ import scala.concurrent.duration._ class CentaurOperationsSpec extends AnyFlatSpec with Matchers { behavior of "validateMetadataJson" - val placeholderSubmittedWorkflow: SubmittedWorkflow = SubmittedWorkflow(id = WorkflowId(UUID.randomUUID()), null, null) - val placeholderWorkflow: Workflow = Workflow(testName = "", null, null, null, null, null, false, false, false, null, null) + val placeholderSubmittedWorkflow: SubmittedWorkflow = + SubmittedWorkflow(id = WorkflowId(UUID.randomUUID()), null, null) + val placeholderWorkflow: Workflow = + Workflow(testName = "", null, null, null, null, null, false, false, false, null, null) val allowableOneWordAdditions = List("farmer") def runTest(json1: String, json2: String, expectMatching: Boolean): Unit = { - val validation = Operations.validateMetadataJson("", - json1.parseJson.asJsObject, - json2.parseJson.asJsObject, - placeholderSubmittedWorkflow, - placeholderWorkflow, - allowableAddedOneWordFields = allowableOneWordAdditions).unsafeToFuture() + val validation = Operations + .validateMetadataJson( + "", + json1.parseJson.asJsObject, + json2.parseJson.asJsObject, + placeholderSubmittedWorkflow, + placeholderWorkflow, + allowableAddedOneWordFields = allowableOneWordAdditions + ) + .unsafeToFuture() Await.ready(validation, atMost = 10.seconds) validation.value.get match { case Success(()) if expectMatching => // great case Success(_) if !expectMatching => fail("Metadata unexpectedly matches") - case Failure(e) if expectMatching => fail("Metadata unexpectedly mismatches", e) + case Failure(e) if expectMatching => fail("Metadata unexpectedly mismatches", e) case Failure(_) if !expectMatching => // great case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") } diff --git a/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala b/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala index 2476bb4b231..0be16d9a57a 100644 --- a/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala +++ b/centaur/src/test/scala/centaur/test/metadata/CallAttemptFailureSpec.scala @@ -6,7 +6,6 @@ import io.circe.ParsingFailure import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class CallAttemptFailureSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "CallAttemptFailure" diff --git a/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala b/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala index 2b9cb409352..f4ce7300385 100644 --- a/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala +++ b/centaur/src/test/scala/centaur/test/metadata/ExtractJobManagerStyleMetadataFieldsSpec.scala @@ -6,7 +6,6 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class ExtractJobManagerStyleMetadataFieldsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "extracting Job Manager style metadata fields" @@ -45,6 +44,8 @@ class ExtractJobManagerStyleMetadataFieldsSpec extends AnyFlatSpec with Cromwell | } |}""".stripMargin - Operations.extractJmStyleMetadataFields(originalMetadata.parseJson.asJsObject) should be(expectedExpectedMetadata.parseJson.asJsObject) + Operations.extractJmStyleMetadataFields(originalMetadata.parseJson.asJsObject) should be( + expectedExpectedMetadata.parseJson.asJsObject + ) } } diff --git a/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala b/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala index f389192aa7f..0f9984d882e 100644 --- a/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala +++ b/centaur/src/test/scala/centaur/testfilecheck/FileCheckerSpec.scala @@ -11,7 +11,6 @@ import software.amazon.awssdk.services.s3.S3Client import software.amazon.awssdk.services.s3.model.{ListObjectsRequest, ListObjectsResponse, S3Object} import org.scalatest.flatspec.AnyFlatSpec - class FileCheckerSpec extends AnyFlatSpec with CromwellTimeoutSpec with MockSugar { import centaur.test.ObjectCounterInstances._ @@ -26,24 +25,29 @@ class FileCheckerSpec extends AnyFlatSpec with CromwellTimeoutSpec with MockSuga private val wrongBucketPrefix = "s3Bucket://my-not-so-cool-bucket/somelogs/empty" private val EmptyTestPath = "" private val testGsPath = "gs://my-cool-bucket/path/to/file" - private val objResponse = ListObjectsResponse.builder() - .contents(util.Arrays.asList(S3Object.builder() - .build())) + private val objResponse = ListObjectsResponse + .builder() + .contents( + util.Arrays.asList( + S3Object + .builder() + .build() + ) + ) .build() private val objRequest = ListObjectsRequest.builder().bucket(bucketName).prefix(dirName).build() private val awsS3Path = awsS3ObjectCounter.parsePath(s3PrefixRegex)(testPath) private val gsPath = gcsObjectCounter.parsePath(gsPrefixRegex)(testGsPath) - "parsePath" should "return a bucket and directories" in { assert(awsS3Path.bucket == bucketName) assert(awsS3Path.directory == dirName) } "parsePath" should "throw Exception for wrong path" in { - assertThrows[centaur.test.IllegalPathException] {awsS3ObjectCounter.parsePath(s3PrefixRegex)(wrongBucketPrefix)} - assertThrows[centaur.test.IllegalPathException] {awsS3ObjectCounter.parsePath(s3PrefixRegex)(testGsPath)} - assertThrows[centaur.test.IllegalPathException] {awsS3ObjectCounter.parsePath(s3PrefixRegex)(EmptyTestPath)} + assertThrows[centaur.test.IllegalPathException](awsS3ObjectCounter.parsePath(s3PrefixRegex)(wrongBucketPrefix)) + assertThrows[centaur.test.IllegalPathException](awsS3ObjectCounter.parsePath(s3PrefixRegex)(testGsPath)) + assertThrows[centaur.test.IllegalPathException](awsS3ObjectCounter.parsePath(s3PrefixRegex)(EmptyTestPath)) } "countObjectAtPath" should "should return 1 if the file exist" in { diff --git a/centaur/test_cromwell.sh b/centaur/test_cromwell.sh index 2f330f1feb0..9161f2da5ad 100755 --- a/centaur/test_cromwell.sh +++ b/centaur/test_cromwell.sh @@ -37,7 +37,7 @@ Arguments: INITIAL_DIR=$(pwd) RUN_DIR=$(pwd) LOG_DIR="${RUN_DIR}"/logs -TEST_THREAD_COUNT=16 +TEST_THREAD_COUNT=16 # Note that most users of this script override this value CENTAUR_SBT_COVERAGE=false CROMWELL_TIMEOUT=10s SUITE="" diff --git a/centaurCwlRunner/src/bin/centaur-cwl-runner.bash b/centaurCwlRunner/src/bin/centaur-cwl-runner.bash deleted file mode 100755 index 8f39c3ec130..00000000000 --- a/centaurCwlRunner/src/bin/centaur-cwl-runner.bash +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env bash - -# `sbt assembly` must have already been run. -build_root="$( dirname "${BASH_SOURCE[0]}" )/../../.." -centaur_cwl_jar="${CENTAUR_CWL_JAR:-"$( \ - find "${build_root}/centaurCwlRunner/target/scala-2.13" -name 'centaur-cwl-runner-*.jar' -print0 \ - | xargs -0 ls -1 -t \ - | head -n 1 \ - )"}" -centaur_cwl_skip_file="${build_root}/centaurCwlRunner/src/main/resources/skipped_tests.csv" - -centaur_cwl_java_args=("-Xmx1g") -if [[ -n "${CENTAUR_CWL_JAVA_ARGS-}" ]]; then - # Allow splitting on space to simulate an exported array - # https://stackoverflow.com/questions/5564418/exporting-an-array-in-bash-script#answer-5564589 - # shellcheck disable=SC2206 - centaur_cwl_java_args+=(${CENTAUR_CWL_JAVA_ARGS}) -fi - -# Handle empty arrays in older versions of bash -# https://stackoverflow.com/questions/7577052/bash-empty-array-expansion-with-set-u#answer-7577209 -java \ - ${centaur_cwl_java_args[@]+"${centaur_cwl_java_args[@]}"} \ - -jar "${centaur_cwl_jar}" \ - --skip-file "${centaur_cwl_skip_file}" \ - "$@" diff --git a/centaurCwlRunner/src/main/resources/application.conf b/centaurCwlRunner/src/main/resources/application.conf deleted file mode 100644 index fcdd3da9c57..00000000000 --- a/centaurCwlRunner/src/main/resources/application.conf +++ /dev/null @@ -1,23 +0,0 @@ -akka { - loggers = ["akka.event.slf4j.Slf4jLogger"] - logging-filter = "akka.event.slf4j.Slf4jLoggingFilter" - - # Silence Akka-Http warning logging of: - # [WARN] [01/28/2019 06:50:04.293] [centaur-acting-like-a-system-akka.actor.default-dispatcher-3] [centaur-acting-like-a-system/Pool(shared->http://localhost:8000)] Connection attempt failed. Backing off new connection attempts for at least 100 milliseconds. - # - # via: - # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/resources/reference.conf#L41-L44 - # https://github.com/akka/akka/blob/v2.5.19/akka-actor/src/main/scala/akka/event/Logging.scala#L377 - # https://github.com/akka/akka-http/blob/v10.1.7/akka-http-core/src/main/scala/akka/http/impl/engine/client/pool/NewHostConnectionPool.scala#L134 - stdout-loglevel = "ERROR" -} - -centaur { - # When running the tests in parallel on a Travis instance we need a bit more time for a reply to come back. - sendReceiveTimeout: 1 minute -} - -cwltool-runner { - # When running the tests in parallel it takes a while for each JVM to initialize Heterodon. Instead use the process. - class = "cwl.CwltoolProcess" -} diff --git a/centaurCwlRunner/src/main/resources/logback.xml b/centaurCwlRunner/src/main/resources/logback.xml deleted file mode 100644 index d92bac61949..00000000000 --- a/centaurCwlRunner/src/main/resources/logback.xml +++ /dev/null @@ -1,48 +0,0 @@ - - - - %d{HH:mm:ss.SSS} [%thread] %-5level %logger - %msg%n - - - - - - - - - - - - - - - - - - - - diff --git a/centaurCwlRunner/src/main/resources/reference.conf b/centaurCwlRunner/src/main/resources/reference.conf deleted file mode 100644 index a906e824372..00000000000 --- a/centaurCwlRunner/src/main/resources/reference.conf +++ /dev/null @@ -1,29 +0,0 @@ -centaur { - cwl-runner { - mode = local - - # End of actual references and begin BA-6546 exceptions... - # Not sure if this reference conf is used by testers outside of DSP/broad? - # If you know for sure then do some combination of: - # - update this comment with the location of those tests - # - update the external tests to explicitly pass in their own config - # - if the external tests are 100% confirmed not to exist state how you know that - # Not really reference, but leaving just in case someone is relying on these values. - papi.default-input-gcs-prefix = "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/" - tesk.default-input-ftp-prefix = "ftp://ftp.hexdump.org/centaur-cwl-conformance/cwl-inputs/" - google { - application-name = "centaur-cwl-runner" - genomics.endpoint-url = "https://lifesciences.googleapis.com/" - genomics.location = "us-central1" - max-attempts = 3 - auth = "application-default" - auths = [ - { - name = "application-default" - scheme = "application_default" - } - ] - } - # End BA-6546 exceptions - } -} diff --git a/centaurCwlRunner/src/main/resources/skipped_tests.csv b/centaurCwlRunner/src/main/resources/skipped_tests.csv deleted file mode 100644 index 08b6b117906..00000000000 --- a/centaurCwlRunner/src/main/resources/skipped_tests.csv +++ /dev/null @@ -1,2 +0,0 @@ -vf-concat.cwl,empty.json -dir5.cwl,dir-job.yml diff --git a/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunner.scala b/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunner.scala deleted file mode 100644 index ec5dcad537e..00000000000 --- a/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunner.scala +++ /dev/null @@ -1,255 +0,0 @@ -package centaur.cwl - -import better.files._ -import cats.effect.IO -import centaur.api.CentaurCromwellClient -import centaur.cwl.Outputs._ -import centaur.test.TestOptions -import centaur.test.standard.{CentaurTestCase, CentaurTestFormat} -import centaur.test.submit.{SubmitHttpResponse, SubmitWorkflowResponse} -import centaur.test.workflow.{AllBackendsRequired, SubmittedWorkflowTracker, Workflow, WorkflowData} -import com.typesafe.scalalogging.StrictLogging -import common.util.VersionUtil -import cromwell.api.model.{Aborted, Failed, NonTerminalStatus, Succeeded} -import cromwell.core.WorkflowOptions -import cromwell.core.path.PathBuilderFactory -import cromwell.core.path.BetterFileMethods.Cmds -import cwl.preprocessor.{CwlFileReference, CwlPreProcessor} -import spray.json._ - -import scala.concurrent.Await -import scala.concurrent.duration.Duration - -/** - * Runs workflows in a "cwl-runner" friendly way. - * - * https://github.com/broadinstitute/cromwell/issues/2590 - * https://github.com/common-workflow-language/common-workflow-language/blob/v1.0.1/CONFORMANCE_TESTS.md - * https://github.com/common-workflow-language/common-workflow-language/blob/v1.0.1/draft-3/cwl-runner.cwl#L5-L68 - * https://github.com/common-workflow-language/common-workflow-language/pull/278/files#diff-ee814a9c027fc9750beb075c283a973cR49 - */ -object CentaurCwlRunner extends StrictLogging { - - case class SkippedTest(sourceName: String, inputsName: String) { - def shouldSkip(args: CommandLineArguments) = { - args.workflowSource.exists(_.name.equalsIgnoreCase(sourceName)) && - args.workflowInputs.exists(_.name.equalsIgnoreCase(inputsName)) - } - } - - case class CommandLineArguments(workflowSource: Option[File] = None, - workflowInputs: Option[File] = None, - quiet: Boolean = false, - outdir: Option[File] = None, - skipFile: Option[File] = None) - - // TODO: This would be cleaner with Enumeratum - object ExitCode extends Enumeration { - - protected case class ExitVal(status: Int) extends super.Val - - implicit class ValueToVal(val exitCodeValue: Value) extends AnyVal { - def status: Int = exitCodeValue.asInstanceOf[ExitVal].status - } - - val Success = ExitVal(0) - val Failure = ExitVal(1) - val NotImplemented = ExitVal(33) - } - - private val cwlPreProcessor = new CwlPreProcessor() - private val centaurCwlRunnerRunMode = CentaurCwlRunnerRunMode.fromConfig(CentaurCwlRunnerConfig.conf) - private val parser = buildParser() - private lazy val centaurCwlRunnerVersion = VersionUtil.getVersion("centaur-cwl-runner") - private lazy val versionString = s"$centaurCwlRunnerVersion ${centaurCwlRunnerRunMode.description}" - - private def showUsage(): ExitCode.Value = { - System.err.println(parser.usage) - ExitCode.Failure - } - - private def buildParser(): scopt.OptionParser[CommandLineArguments] = { - new scopt.OptionParser[CommandLineArguments]("java -jar /path/to/centaurCwlRunner.jar") { - head("centaur-cwl-runner", versionString) - - help("help").text("Centaur CWL Runner - Cromwell integration testing environment") - - version("version").text("Print version and exit") - - arg[String]("workflow-source").text("Workflow source file.").required(). - action((s, c) => c.copy(workflowSource = Option(File(s)))) - - arg[String]("inputs").text("Workflow inputs file.").optional(). - action((s, c) => c.copy(workflowInputs = Option(File(s)))) - - opt[Unit]("quiet").text("Only print warnings and errors.").optional(). - action((_, c) => c.copy(quiet = true)) - - opt[String]("outdir").text("Output directory, default current directory.").optional(). - action((s, c) => - c.copy(outdir = Option(File(s)))) - - opt[String]("skip-file").text("A csv file describing workflows to be skipped based on their name.").optional(). - action((s, c) => - c.copy(skipFile = Option(File(s)))) - } - } - - private def runCentaur(args: CommandLineArguments): ExitCode.Value = { - - def zipSiblings(file: File): File = { - val zipFile = File.newTemporaryFile("cwl_imports.", ".zip") - val dir = file.parent - if (!args.quiet) { - logger.info(s"Zipping files under 1mb in $dir to $zipFile") - } - // TODO: Only include files under 1mb for now. When cwl runners run in parallel this can use a lot of space. - val files = dir - .children - .filter(_.isRegularFile) - .filter(_.size < 1 * 1024 * 1024) - Cmds.zip(files.toSeq: _*)(zipFile) - zipFile - } - - val workflowPath = args.workflowSource.get - val (parsedWorkflowPath, workflowRoot) = workflowPath.path.toAbsolutePath.toString.split("#") match { - case Array(file) => File(file) -> None - case Array(file, root) => File(file) -> Option(root) - } - val outdirOption = args.outdir.map(_.pathAsString) - val testName = workflowPath.name - val preProcessedWorkflow = cwlPreProcessor - .preProcessCwlToString(CwlFileReference(parsedWorkflowPath, workflowRoot)) - .value.unsafeRunSync() match { - case Left(errors) => - logger.error(s"Failed to pre process cwl workflow: ${errors.toList.mkString(", ")}") - return ExitCode.Failure - case Right(v) => v - } - - val workflowContents = centaurCwlRunnerRunMode.preProcessWorkflow(preProcessedWorkflow) - val inputContents = args.workflowInputs - .map(centaurCwlRunnerRunMode.preProcessInput) - .map(preProcessed => { - preProcessed.value.unsafeRunSync() match { - case Left(errors) => - logger.error(s"Failed to pre process cwl workflow: ${errors.toList.mkString(", ")}") - return ExitCode.Failure - case Right(value) => value - } - }) - - val workflowType = Option("cwl") - val workflowTypeVersion = None - val optionsContents = outdirOption map { outdir => - JsObject("cwl_outdir" -> JsString(outdir)).compactPrint - } - val labels = List.empty - val zippedImports = Option(zipSiblings(workflowPath)) // TODO: Zipping all the things! Be more selective. - val backends = AllBackendsRequired(List.empty) - val workflowMetadata = None - val notInMetadata = List.empty - val directoryContentCounts = None - val testFormat = CentaurTestFormat.WorkflowSuccessTest - val testOptions = TestOptions(List.empty, ignore = false) - val submitResponseOption = None - - val submittedWorkflowTracker = new SubmittedWorkflowTracker() - - val workflowData = WorkflowData( - Option(workflowContents), - None, - None, - workflowType, - workflowTypeVersion, - inputContents.map(IO.pure), - optionsContents.map(IO.pure), - labels, - zippedImports - ) - val workflow = Workflow( - testName, - workflowData, - workflowMetadata, - notInMetadata, - directoryContentCounts, - backends, - retryTestFailures = false, - allowOtherOutputs = true, - skipDescribeEndpointValidation = true, - submittedWorkflowTracker = submittedWorkflowTracker, - maximumAllowedTime = None - ) - - val testCase = CentaurTestCase(workflow, testFormat, testOptions, submittedWorkflowTracker, submitResponseOption)(cromwellTracker = None) - - if (!args.quiet) { - logger.info(s"Starting test for $workflowPath") - } - - val pathBuilderFactory: PathBuilderFactory = centaurCwlRunnerRunMode.pathBuilderFactory - - try { - import CentaurCromwellClient.{blockingEc, system} - lazy val pathBuilder = Await.result(pathBuilderFactory.withOptions(WorkflowOptions.empty), Duration.Inf) - - testCase.testFunction.run.unsafeRunSync() match { - case unexpected: SubmitHttpResponse => - logger.error(s"Unexpected response: $unexpected") - ExitCode.Failure - case SubmitWorkflowResponse(submittedWorkflow) => - val status = CentaurCromwellClient.status(submittedWorkflow).unsafeRunSync() - status match { - case unexpected: NonTerminalStatus => - logger.error(s"Unexpected status: $unexpected") - ExitCode.Failure - case Aborted => - logger.error(s"Unexpected abort.") - ExitCode.Failure - case Failed => - logger.error(s"Unexpected failure.") - ExitCode.Failure - case Succeeded => - val result = handleOutput(submittedWorkflow, pathBuilder) - if (!args.quiet) { - logger.info(s"Result: $result") - } else { - // Print directly to stdout during --quiet - println(result) - } - ExitCode.Success - } - } - } finally { - zippedImports map { zipFile => - if (!args.quiet) { - logger.info(s"Deleting $zipFile") - } - zipFile.delete(swallowIOExceptions = true) - } - Await.result(CentaurCromwellClient.system.terminate(), Duration.Inf) - () - } - } - - private def skip(args: CommandLineArguments): Boolean = - args.skipFile - .map(_.lines).getOrElse(List.empty) - .map(_.split(',')) - .map({ - case Array(source, inputs) => SkippedTest(source, inputs) - case invalid => throw new RuntimeException(s"Invalid skipped_tests file: $invalid") - }) - .exists(_.shouldSkip(args)) - - def main(args: Array[String]): Unit = { - val parsedArgsOption = parser.parse(args, CommandLineArguments()) - val exitCode = parsedArgsOption match { - case Some(parsedArgs) if skip(parsedArgs) => ExitCode.NotImplemented - case Some(parsedArgs) => runCentaur(parsedArgs) - case None => showUsage() - } - System.exit(exitCode.status) - } -} diff --git a/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunnerConfig.scala b/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunnerConfig.scala deleted file mode 100644 index 0123efd7b8d..00000000000 --- a/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunnerConfig.scala +++ /dev/null @@ -1,7 +0,0 @@ -package centaur.cwl - -import centaur.CentaurConfig - -object CentaurCwlRunnerConfig { - lazy val conf = CentaurConfig.conf.getConfig("cwl-runner") -} diff --git a/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunnerRunMode.scala b/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunnerRunMode.scala deleted file mode 100644 index 83b6626c983..00000000000 --- a/centaurCwlRunner/src/main/scala/centaur/cwl/CentaurCwlRunnerRunMode.scala +++ /dev/null @@ -1,94 +0,0 @@ -package centaur.cwl - -import better.files.File -import cloud.nio.impl.ftp.FtpFileSystems -import com.typesafe.config.Config -import common.validation.IOChecked._ -import cromwell.core.path.Obsolete.Paths -import cromwell.core.path.{DefaultPathBuilderFactory, PathBuilderFactory} -import cromwell.filesystems.ftp.{CromwellFtpFileSystems, FtpPathBuilderFactory} -import cromwell.filesystems.gcs.GcsPathBuilderFactory -import cwl.preprocessor.CwlPreProcessor - -sealed trait CentaurCwlRunnerRunMode { - /** - * Returns a user friendly of this run mode. - */ - def description: String - - /** - * Returns a factory that can create a path builder for this run mode. - */ - def pathBuilderFactory: PathBuilderFactory - - /** - * Runs any preprocessing as needed on a workflow. - * - * For example, may prefix relative paths so that absolute URLs are used. - */ - def preProcessWorkflow(workflow: String): String = workflow - - /** - * Runs any preprocessing as needed on inputs. - * - * For example, may prefix relative paths so that absolute URLs are used. - */ - def preProcessInput(path: File): IOChecked[String] = path.contentAsString.validIOChecked -} - -object CentaurCwlRunnerRunMode { - def fromConfig(conf: Config): CentaurCwlRunnerRunMode = { - conf.getString("mode") match { - case "local" => LocalRunMode - case "papi_v2alpha1" | "papi_v2beta" => PapiRunMode(conf) - case "tesk" => TeskRunMode(conf) - case unknown => throw new UnsupportedOperationException(s"mode not recognized: $unknown") - } - } -} - -case object LocalRunMode extends CentaurCwlRunnerRunMode { - val cwlPreProcessor = new CwlPreProcessor() - override lazy val description: String = "local" - override lazy val pathBuilderFactory: PathBuilderFactory = DefaultPathBuilderFactory - /* - * If the file is a relative local path, resolve it against the path of the input json. - */ - private def inputFilesMapper(inputJsonPath: File)(file: String) = { - if (!Paths.get(file).isAbsolute) inputJsonPath.sibling(file).toString - else file - } - - override def preProcessInput(input: File): IOChecked[String] = cwlPreProcessor.preProcessInputFiles(input.contentAsString, inputFilesMapper(input)) -} - -case class PapiRunMode(conf: Config) extends CentaurCwlRunnerRunMode { - private lazy val googleConfig = conf.getConfig("google") - private lazy val preprocessor = new CloudPreprocessor(conf, "papi.default-input-gcs-prefix") - - override lazy val description: String = s"papi ${googleConfig.getString("auth")}" - - override lazy val pathBuilderFactory: PathBuilderFactory = { - GcsPathBuilderFactory(conf, googleConfig) - } - - override def preProcessWorkflow(workflow: String): String = preprocessor.preProcessWorkflow(workflow) - - override def preProcessInput(input: File): IOChecked[String] = preprocessor.preProcessInput(input.contentAsString) -} - -case class TeskRunMode(conf: Config) extends CentaurCwlRunnerRunMode { - private lazy val ftpConfig = conf.getConfig("ftp") - private lazy val preprocessor = new CloudPreprocessor(conf, "tesk.default-input-ftp-prefix") - - override lazy val description: String = "tesk" - - override lazy val pathBuilderFactory: PathBuilderFactory = { - val fileSystemsConfiguration = FtpFileSystems.DefaultConfig.copy(capacity = 2) - new FtpPathBuilderFactory(conf, ftpConfig, new CromwellFtpFileSystems(new FtpFileSystems(fileSystemsConfiguration))) - } - - override def preProcessWorkflow(workflow: String): String = preprocessor.preProcessWorkflow(workflow) - - override def preProcessInput(input: File): IOChecked[String] = preprocessor.preProcessInput(input.contentAsString) -} diff --git a/centaurCwlRunner/src/main/scala/centaur/cwl/CloudPreprocessor.scala b/centaurCwlRunner/src/main/scala/centaur/cwl/CloudPreprocessor.scala deleted file mode 100644 index 68e79b9b701..00000000000 --- a/centaurCwlRunner/src/main/scala/centaur/cwl/CloudPreprocessor.scala +++ /dev/null @@ -1,102 +0,0 @@ -package centaur.cwl -import better.files.File -import com.typesafe.config.Config -import common.util.StringUtil._ -import common.validation.IOChecked.IOChecked -import cwl.preprocessor.CwlPreProcessor -import io.circe.optics.JsonPath -import io.circe.optics.JsonPath._ -import io.circe.yaml.Printer.StringStyle -import io.circe.{Json, yaml} -import net.ceedubs.ficus.Ficus._ -import wom.util.YamlUtils - -/** - * Tools to pre-process the CWL workflows and inputs before feeding them to Cromwell so they can be executed on PAPI. - */ -class CloudPreprocessor(config: Config, prefixConfigPath: String) { - val cwlPreProcessor = new CwlPreProcessor() - - // Cloud directory where inputs for conformance tests are stored - private val cloudPrefix = config.as[String](prefixConfigPath).ensureSlashed - - // Default docker pull image - val DefaultDockerPull = "dockerPull" -> Json.fromString("ubuntu:latest") - - // Default docker image to be injected in a pre-existing requirements array - private val DefaultDockerHint: Json = { - Json.obj( - "class" -> Json.fromString("DockerRequirement"), - DefaultDockerPull - ) - } - - // hints array with default docker requirement - private val DefaultDockerHintList: Json = { - Json.obj( - "hints" -> Json.arr(DefaultDockerHint) - ) - } - - // Parse value, apply f to it, and print it back to String using the printer - private def process(value: String, f: Json => Json, printer: Json => String) = { - YamlUtils.parse(value) match { - case Left(error) => throw new Exception(error.getMessage) - case Right(json) => printer(f(json)) - } - } - - // Process and print back as YAML - private def processYaml(value: String)(f: Json => Json) = - process(value, f, yaml.Printer.spaces2.copy(stringStyle = StringStyle.DoubleQuoted).pretty) - - // Prefix the string at "key" with the cloud prefix - private def prefixLocation(value: String): String = { - cloudPrefix + File(value.stripPrefix("file://")).name - } - - // Function to check if the given json has the provided key / value pair - private def hasKeyValue(key: String, value: String): Json => Boolean = { - root.selectDynamic(key).string.exist(_.equalsIgnoreCase(value)) - } - - /** - * Pre-process input file by prefixing all files and directories with the cloud prefix - */ - def preProcessInput(input: String): IOChecked[String] = cwlPreProcessor.preProcessInputFiles(input, prefixLocation) - - // Check if the given path (as an array or object) has a DockerRequirement element - def hasDocker(jsonPath: JsonPath)(json: Json): Boolean = { - val hasDockerInArray: Json => Boolean = jsonPath.arr.exist(_.exists(hasKeyValue("class", "DockerRequirement"))) - val hasDockerInObject: Json => Boolean = jsonPath.obj.exist(_.kleisli("DockerRequirement").nonEmpty) - - hasDockerInArray(json) || hasDockerInObject(json) - } - - // Check if the given Json has a docker image in hints or requirements - def hasDocker(json: Json): Boolean = hasDocker(root.hints)(json) || hasDocker(root.requirements)(json) - - // Add a default docker hint to the workflow if it doesn't have one - private val addDefaultDocker: Json => Json = workflow => if (!hasDocker(workflow)) { - /* - * deepMerge does not combine objects together but replaces keys which would overwrite existing hints - * so first check if there are hints already and if so add our docker one. - * Also turns out that the hints section can be either an array or an object. - * When it gets saladed the object is transformed to an array but because we deal with unsaladed cwl here - * we have to handle both cases. - */ - val hintsAsArray = root.hints.arr.modifyOption(_ :+ DefaultDockerHint)(workflow) - val hintsAsObject = root.hints.obj.modifyOption(_.add("DockerRequirement", Json.obj(DefaultDockerPull)))(workflow) - - hintsAsArray - .orElse(hintsAsObject) - .getOrElse(workflow.deepMerge(DefaultDockerHintList)) - } else workflow - - private val prefixDefaultFilesInCwl = CwlPreProcessor.mapFilesAndDirectories(prefixLocation) _ - - /** - * Pre-process the workflow by adding a default docker hint iff it doesn't have one - */ - def preProcessWorkflow(workflow: String): String = processYaml(workflow)(addDefaultDocker.andThen(prefixDefaultFilesInCwl)) -} diff --git a/centaurCwlRunner/src/main/scala/centaur/cwl/OutputManipulator.scala b/centaurCwlRunner/src/main/scala/centaur/cwl/OutputManipulator.scala deleted file mode 100644 index dcbe3a70ec8..00000000000 --- a/centaurCwlRunner/src/main/scala/centaur/cwl/OutputManipulator.scala +++ /dev/null @@ -1,237 +0,0 @@ -package centaur.cwl - -import common.util.StringUtil._ -import cromwell.core.path.{Path, PathBuilder} -import cwl.CwlCodecs._ -import cwl.command.ParentName -import cwl.ontology.Schema -import cwl.{File => CwlFile, _} -import io.circe.literal._ -import io.circe.syntax._ -import io.circe.{Json, JsonObject} -import mouse.all._ -import org.apache.commons.codec.digest.DigestUtils -import shapeless.{Inl, Poly1} -import spray.json.{JsArray, JsBoolean, JsNull, JsNumber, JsObject, JsString, JsValue} - -//Take cromwell's outputs and format them as expected by the spec -object OutputManipulator extends Poly1 { - - //In an Ideal world I'd return a Coproduct of these types and leave the asJson-ing to the handleOutput - def resolveOutput(jsValue: JsValue, - pathBuilder: PathBuilder, - mot: MyriadOutputType, - schemaOption: Option[Schema]): Json = { - mot.fold(this).apply(jsValue, pathBuilder, schemaOption) - } - - private def hashFile(path: Path): Option[String] = path.exists.option("sha1$" + path.sha1.toLowerCase) - private def sizeFile(path: Path): Option[Long] = path.exists.option(path.size) - - private def stringToFile(pathAsString: String, pathBuilder: PathBuilder): Json = { - val path = pathBuilder.build(pathAsString).get - - if (path.exists()) { - CwlFile( - location = Option(path.name), - checksum = hashFile(path), - size = sizeFile(path) - ).asJson - } else { - Json.Null - } - } - - private def populateFileFields(pathBuilder: PathBuilder, - schemaOption: Option[Schema] - )(obj: JsonObject): JsonObject = { - val path = pathBuilder.build(obj.kleisli("location").get.asString.get).get - val isFile = obj.kleisli("class").exists(_.asString.contains("File")) - val isDirectory = obj.kleisli("class").exists(_.asString.contains("Directory")) - - // Return the default value if the value at key is either absent or null, otherwise return the value - def valueOrNull(key: String, default: Json) = obj.kleisli(key) match { - case Some(Json.Null) => default - case Some(other) => other - case None => default - } - - def populateInnerFiles(json: Json): Option[Json] = { - - def populateInnerFile(file: Json): Json = { - file.asObject - .map(populateFileFields(pathBuilder, schemaOption)) - .map(Json.fromJsonObject) - .orElse(file.asString.map(stringToFile(_, pathBuilder))) - .getOrElse(file) - } - - // Assume the json is an array ("secondaryFiles" and "listing" are both arrays). - val innerFiles = json.asArray.get - // Remove any files that don't actually exist. - val filteredFiles = innerFiles.map(populateInnerFile).filterNot(_.isNull) - // The cwl test runner doesn't expect a "secondaryFiles" or "listing" field at all if it's empty. - if (filteredFiles.nonEmpty) Option(Json.arr(filteredFiles: _*)) else None - } - - def updateFileOrDirectoryWithNestedFiles(obj: JsonObject, fieldName: String) = { - // Cromwell metadata has a field for all values even if their content is empty - // remove it as the cwl test runner expects nothing instead - val withoutField = obj.remove(fieldName) - - // If the field was not empty, add it back with each inner file / directory properly updated as well - populateInnerFiles(obj.kleisli(fieldName).get) - .map(withoutField.add(fieldName, _)) - .getOrElse(withoutField) - } - - // Get sha1 from the content field - // We need this because some task return a "File" object that actually does not exist on disk but has its content provided directly instead - def hashContent: Option[String] = obj.kleisli("contents").flatMap(_.asString.map(DigestUtils.sha1Hex).map("sha1$" + _)) - - // Get size from the content field - // We need this because some task return a "File" object that actually does not exist on disk but has its content provided directly instead - def sizeContent: Option[Long] = obj.kleisli("contents").flatMap(_.asString.map(_.length.toLong)) - - // "as a special case, cwltest only matches the trailing part of location in the output sections so that it - // can do something reasonable regardless of URL scheme or prefix" - Peter Amstutz, 2019-04-16, CWL gitter - val updatedLocation = obj.add("location", Json.fromString(path.pathAsString)) - - // Get the format - def formatOption: Option[String] = for { - klesliFormat <- obj.kleisli("format") - format <- klesliFormat.asString - } yield format - - // Lookup the full format using the namespaces - def fullFormatOption: Option[String] = for { - format <- formatOption - schema <- schemaOption - } yield schema.fullIri(format) - - if (isFile) { - /* - * In order of priority use: - * 1) the checksum value in the metadata - * 2) the checksum calculated from the value of the "contents" field of the metadata - * 3) the checksum calculated from the file itself - */ - val defaultChecksum = hashContent.orElse(hashFile(path)).map(Json.fromString).getOrElse(Json.Null) - val checksum = valueOrNull("checksum", defaultChecksum) - - /* - * In order of priority use: - * 1) the size value in the metadata - * 2) the size calculated from the value of the "contents" field of the metadata - * 3) the size calculated from the file itself - */ - val defaultSize = sizeContent.orElse(sizeFile(path)).map(Json.fromLong).getOrElse(Json.Null) - val size = valueOrNull("size", defaultSize) - - val basename: Option[Json] = - Option(valueOrNull("basename", path.exists.option(path.name).map(Json.fromString).getOrElse(Json.Null))) - - /* - In order of priority use: - 1) the full format if $namespaces are available - 2) the original format - */ - val defaultFormatOption: Option[Json] = fullFormatOption.orElse(formatOption).map(Json.fromString) - - val withChecksumAndSize = updatedLocation - .add("checksum", checksum) - .add("size", size) - // conformance test does not expect "contents" in the output - .remove("contents") - - val withBasename = basename - .map(withChecksumAndSize.add("basename", _)) - .getOrElse(withChecksumAndSize) - - val withFormat = defaultFormatOption - .map(withBasename.add("format", _)) - .getOrElse(withBasename) - - updateFileOrDirectoryWithNestedFiles(withFormat, "secondaryFiles") - } else if (isDirectory) { - updateFileOrDirectoryWithNestedFiles(updatedLocation, "listing"). - add("basename", path.nameWithoutExtension |> Json.fromString) - } else throw new RuntimeException(s"${path.pathAsString} is neither a valid file or a directory") - } - - private def resolveOutputViaInnerType(moits: Array[MyriadOutputInnerType]) - (jsValue: JsValue, - pathBuilder: PathBuilder, - schemaOption: Option[Schema]): Json = { - def sprayJsonToCirce(jsValue: JsValue): Json = { - import io.circe.parser._ - val jsonString = jsValue.compactPrint - parse(jsonString).getOrElse( - sys.error(s"Failed to parse Json output as Json... something is very wrong: '$jsonString'") - ) - } - - (jsValue, moits) match { - //CWL expects quite a few enhancements to the File structure, hence... - case (JsString(metadata), Array(Inl(CwlType.File))) => stringToFile(metadata, pathBuilder) - // If it's a JsObject it means it's already in the right format, we just want to fill in some values that might not - // have been populated like "checksum" and "size" - case (obj: JsObject, a) if a.contains(Inl(CwlType.File)) || a.contains(Inl(CwlType.Directory)) => - val json: Json = sprayJsonToCirce(obj) - val fileExists = (for { - o <- json.asObject - l <- o.kleisli("location") - s <- l.asString - c = if (a.contains(Inl(CwlType.Directory))) s.ensureSlashed else s - p <- pathBuilder.build(c).toOption - } yield p.exists).getOrElse(false) - if (fileExists) json.mapObject(populateFileFields(pathBuilder, schemaOption)) else Json.Null - case (JsNumber(metadata), Array(Inl(CwlType.Long))) => metadata.longValue.asJson - case (JsNumber(metadata), Array(Inl(CwlType.Float))) => metadata.floatValue.asJson - case (JsNumber(metadata), Array(Inl(CwlType.Double))) => metadata.doubleValue.asJson - case (JsNumber(metadata), Array(Inl(CwlType.Int))) => metadata.intValue.asJson - case (JsString(metadata), Array(Inl(CwlType.String))) => metadata.asJson - - //The Anys. They have to be done for each type so that the asJson can use this type information when going back to Json representation - case (JsString(metadata), Array(Inl(CwlType.Any))) => metadata.asJson - case (JsNumber(metadata), Array(Inl(CwlType.Any))) => metadata.asJson - case (obj: JsObject, Array(Inl(CwlType.Any))) => sprayJsonToCirce(obj) - case (JsBoolean(metadata), Array(Inl(CwlType.Any))) => metadata.asJson - case (array: JsArray, Array(Inl(CwlType.Any))) => sprayJsonToCirce(array) - case (JsNull, a) if a.contains(Inl(CwlType.Any)) || a.contains(Inl(CwlType.Null)) => Json.Null - - case (JsArray(metadata), Array(tpe)) if tpe.select[OutputArraySchema].isDefined => - (for { - schema <- tpe.select[OutputArraySchema] - items = schema.items - innerType <- items.select[MyriadOutputInnerType] - outputJson = metadata.map(m => - resolveOutputViaInnerType(Array(innerType))(m, pathBuilder, schemaOption)).asJson - } yield outputJson).getOrElse(throw new RuntimeException(s"We currently do not support output arrays with ${tpe.select[OutputArraySchema].get.items} inner type")) - case (JsObject(metadata), Array(tpe)) if tpe.select[OutputRecordSchema].isDefined => - def processField(field: OutputRecordField) = { - val parsedName = FullyQualifiedName(field.name)(ParentName.empty).id - field.`type`.select[MyriadOutputInnerType] map { parsedName -> _ } - } - - (for { - schema <- tpe.select[OutputRecordSchema] - fields <- schema.fields - typeMap = fields.flatMap(processField).toMap - outputJson = metadata.map({ - case (k, v) => k -> resolveOutputViaInnerType(Array(typeMap(k)))(v, pathBuilder, schemaOption) - }).asJson - } yield outputJson).getOrElse(throw new RuntimeException(s"We currently do not support output record schemas with ${tpe.select[OutputArraySchema].get.items} inner type")) - case (json, tpe) => throw new RuntimeException(s"We currently do not support outputs (${json.getClass.getSimpleName}) of $json and type $tpe") - } - } - - implicit val moit: Case.Aux[MyriadOutputInnerType, (JsValue, PathBuilder, Option[Schema]) => Json] = at { t => - resolveOutputViaInnerType(Array(t)) - } - - implicit val amoit: Case.Aux[Array[MyriadOutputInnerType], (JsValue, PathBuilder, Option[Schema]) => Json] = - at { amoit => - resolveOutputViaInnerType(amoit) - } -} diff --git a/centaurCwlRunner/src/main/scala/centaur/cwl/Outputs.scala b/centaurCwlRunner/src/main/scala/centaur/cwl/Outputs.scala deleted file mode 100644 index c63d95399d7..00000000000 --- a/centaurCwlRunner/src/main/scala/centaur/cwl/Outputs.scala +++ /dev/null @@ -1,93 +0,0 @@ -package centaur.cwl - -import centaur.api.CentaurCromwellClient -import centaur.test.metadata.WorkflowFlatMetadata._ -import common.collections.EnhancedCollections._ -import common.validation.IOChecked._ -import cromwell.api.model.SubmittedWorkflow -import cromwell.core.path.PathBuilder -import cwl.ontology.Schema -import cwl.{Cwl, CwlDecoder, MyriadOutputType} -import io.circe.Json -import io.circe.syntax._ -import mouse.map._ -import shapeless.Poly1 -import spray.json.{JsObject, JsString, JsValue} - -object Outputs { - - //When the string returned is not valid JSON, it is effectively an exception as CWL runner expects JSON to be returned - def handleOutput(submittedWorkflow: SubmittedWorkflow, pathBuilder: PathBuilder): String = { - val metadata: Map[String, JsValue] = CentaurCromwellClient.metadata(submittedWorkflow).unsafeRunSync().asFlat.value - - // Wrapper function to provide the right signature for `intersectWith` below. - def outputResolver(schemaOption: Option[Schema])(jsValue: JsValue, mot: MyriadOutputType): Json = { - OutputManipulator.resolveOutput(jsValue, pathBuilder, mot, schemaOption) - } - - //Sorry for all the nesting, but spray json JsValue doesn't have optional access methods like Argonaut/circe, - //thus no for comprehensions for us :( - metadata.get("submittedFiles.workflow") match { - case Some(JsString(workflow)) => - - val parseCwl: IOChecked[Cwl] = CwlDecoder.decodeCwlString( - workflow, - submittedWorkflow.workflow.zippedImports, - submittedWorkflow.workflow.workflowRoot - ) - - parseCwl.value.attempt.unsafeRunSync() match { - case Right(Right(cwl)) => - - CentaurCromwellClient.outputs(submittedWorkflow).unsafeRunSync().outputs match { - case JsObject(map) => - val typeMap: Map[String, MyriadOutputType] = cwl.fold(CwlOutputsFold) - val mungeTypeMap = typeMap.mapKeys(stripTypeMapKey) - - val mungeOutputMap = map.mapKeys(stripOutputKey) - - mungeOutputMap. - //This lets us operate on the values of the output values and types for a particular output key - intersectWith(mungeTypeMap)(outputResolver(cwl.schemaOption)). - //converting the whole response to Json using Circe's auto-encoder derivation - asJson. - //drop null values so that we don't print when Option == None - printWith(io.circe.Printer.spaces2.copy(dropNullValues = true)) - case other => s"it seems cromwell is not returning outputs as a Jsobject but is instead a $other" - } - case Right(Left(error)) => s"couldn't parse workflow: $workflow failed with error: $error" - case Left(error) => s"Exception when trying to read workflow: $workflow failed with error: $error" - } - case Some(other) => s"received the value $other when the workflow string was expected" - case None => "the workflow is no longer in the metadata payload, it's a problem" - } - } - - //Ids come out of SALAD pre-processing with a filename prepended. This gets rid of it - // Also gets rid of the parent workflow name if present - def stripTypeMapKey(key: String): String = key - .substring(key.lastIndexOf("#") + 1, key.length) - .split("/").last - - //Ids come out of Cromwell with a prefix, separated by a ".". This takes everything to the right, - //as CWL wants it - def stripOutputKey(key: String): String = key.substring(key.lastIndexOf(".") + 1, key.length) - -} - -object CwlOutputsFold extends Poly1 { - import cwl._ - - implicit def wf: Case.Aux[cwl.Workflow, Map[String, MyriadOutputType]] = at[cwl.Workflow] { - _.outputs.map(output => output.id -> output.`type`.get).toMap - } - - implicit def clt: Case.Aux[cwl.CommandLineTool, Map[String, MyriadOutputType]] = at[cwl.CommandLineTool] { - _.outputs.map(output => output.id -> output.`type`.get).toMap - } - - implicit def et: Case.Aux[cwl.ExpressionTool, Map[String, MyriadOutputType]] = at[cwl.ExpressionTool] { - _.outputs.map(output => output.id -> output.`type`.get).toMap - } -} - diff --git a/centaurCwlRunner/src/test/resources/application.conf b/centaurCwlRunner/src/test/resources/application.conf deleted file mode 100644 index 27c24fa07bf..00000000000 --- a/centaurCwlRunner/src/test/resources/application.conf +++ /dev/null @@ -1 +0,0 @@ -papi.default-input-gcs-prefix="gs://centaur-cwl-conformance-1f501e3/cwl-inputs/" diff --git a/centaurCwlRunner/src/test/scala/CloudPreprocessorSpec.scala b/centaurCwlRunner/src/test/scala/CloudPreprocessorSpec.scala deleted file mode 100644 index df13e49b666..00000000000 --- a/centaurCwlRunner/src/test/scala/CloudPreprocessorSpec.scala +++ /dev/null @@ -1,342 +0,0 @@ -import centaur.cwl.CloudPreprocessor -import com.typesafe.config.ConfigFactory -import common.assertion.CromwellTimeoutSpec -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -import wom.util.YamlUtils - -class CloudPreprocessorSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { - behavior of "PAPIPreProcessor" - - val pAPIPreprocessor = new CloudPreprocessor(ConfigFactory.load(), "papi.default-input-gcs-prefix") - - def validate(result: String, expectation: String) = { - val parsedResult = YamlUtils.parse(result).toOption.get - val parsedExpectation = YamlUtils.parse(expectation).toOption.get - - // This is an actual Json comparison from circe - parsedResult shouldBe parsedExpectation - } - - it should "prefix files and directories in inputs" in { - validate( - pAPIPreprocessor.preProcessInput( - """|{ - | "input": { - | "null": null, - | "file": { - | "location": "whale.txt", - | "class": "File", - | "secondaryFiles": [ - | { - | "class": File, - | "location": "hello.txt" - | } - | ], - | "default": { - | "location": "default_whale.txt", - | "class": "File", - | } - | }, - | "directory": { - | "location": "ref.fasta", - | "class": "Directory", - | "listing": [ - | { - | "class": File, - | "location": "hello.txt" - | } - | ] - | } - | } - |} - |""".stripMargin).value.unsafeRunSync().toOption.get, - """|{ - | "input": { - | "null": null, - | "file": { - | "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/whale.txt", - | "class": "File", - | "secondaryFiles": [ - | { - | "class": File, - | "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/hello.txt" - | } - | ], - | "default": { - | "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/default_whale.txt", - | "class": "File", - | } - | }, - | "directory": { - | "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/ref.fasta", - | "class": "Directory", - | "listing": [ - | { - | "class": File, - | "location": "gs://centaur-cwl-conformance-1f501e3/cwl-inputs/hello.txt" - | } - | ] - | } - | } - |} - |""".stripMargin - ) - } - - it should "prefix files and directories in workflow" in { - validate( - pAPIPreprocessor.preProcessWorkflow( - """|class: CommandLineTool - |cwlVersion: v1.0 - |requirements: - | - class: DockerRequirement - | dockerPull: ubuntu:latest - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | default: - | class: File - | location: args.py - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin), - """|class: CommandLineTool - |cwlVersion: v1.0 - |requirements: - | - class: DockerRequirement - | dockerPull: ubuntu:latest - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | default: - | class: File - | location: gs://centaur-cwl-conformance-1f501e3/cwl-inputs/args.py - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin - ) - } - - it should "add default docker image if there's no hint" in { - validate( - pAPIPreprocessor.preProcessWorkflow( - """|class: CommandLineTool - |cwlVersion: v1.0 - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin), - """|class: CommandLineTool - |hints: - | - class: DockerRequirement - | dockerPull: ubuntu:latest - |cwlVersion: v1.0 - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin - ) - } - - it should "append default docker image to existing hint as an array" in { - validate( - pAPIPreprocessor.preProcessWorkflow( - """|class: CommandLineTool - |hints: - | - class: EnvVarRequirement - | envDef: - | TEST_ENV: $(inputs.in) - |cwlVersion: v1.0 - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin), - """|class: CommandLineTool - |hints: - | - class: EnvVarRequirement - | envDef: - | TEST_ENV: $(inputs.in) - | - class: DockerRequirement - | dockerPull: ubuntu:latest - |cwlVersion: v1.0 - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin - ) - } - - it should "append default docker image to existing hints as an object" in { - validate( - pAPIPreprocessor.preProcessWorkflow( - """|class: CommandLineTool - |cwlVersion: v1.0 - |inputs: - | in: string - |outputs: - | out: - | type: File - | outputBinding: - | glob: out - | - |hints: - | EnvVarRequirement: - | envDef: - | TEST_ENV: $(inputs.in) - | - |baseCommand: ["/bin/bash", "-c", "echo $TEST_ENV"] - | - |stdout: out - |""".stripMargin), - """|class: CommandLineTool - |cwlVersion: v1.0 - |inputs: - | in: string - |outputs: - | out: - | type: File - | outputBinding: - | glob: out - | - |hints: - | EnvVarRequirement: - | envDef: - | TEST_ENV: $(inputs.in) - | DockerRequirement: - | dockerPull: ubuntu:latest - | - |baseCommand: ["/bin/bash", "-c", "echo $TEST_ENV"] - | - |stdout: out - |""".stripMargin - ) - } - - it should "not replace existing docker requirement in an object" in { - val workflow = """|class: CommandLineTool - |cwlVersion: v1.0 - |requirements: - | DockerRequirement: - | dockerPull: python27:slim - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin - validate(pAPIPreprocessor.preProcessWorkflow(workflow), workflow) - } - - it should "not replace existing docker hint in an object" in { - val workflow = """|class: CommandLineTool - |cwlVersion: v1.0 - |hints: - | DockerRequirement: - | dockerPull: python27:slim - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin - validate(pAPIPreprocessor.preProcessWorkflow(workflow), workflow) - } - - it should "not replace existing docker hint in an array" in { - val workflow = """|class: CommandLineTool - |cwlVersion: v1.0 - |hints: - |- class: DockerRequirement - | dockerPull: python27:slim - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin - validate(pAPIPreprocessor.preProcessWorkflow(workflow), workflow) - } - - it should "not replace existing docker requirement in an array" in { - val workflow = """|class: CommandLineTool - |cwlVersion: v1.0 - |requirements: - |- class: DockerRequirement - | dockerPull: python27:slim - |inputs: - | - id: reference - | type: File - | inputBinding: { position: 2 } - | - |outputs: - | args: string[] - | - |baseCommand: python - |arguments: ["bwa", "mem"] - |""".stripMargin - validate(pAPIPreprocessor.preProcessWorkflow(workflow), workflow) - } - - it should "throw an exception if yaml / json can't be parse" in { - val invalid = - """ - |{ [invalid]: } - """.stripMargin - - an[Exception] shouldBe thrownBy(pAPIPreprocessor.preProcessWorkflow(invalid)) - } -} diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala index a1156fc33e9..d4a50238176 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileProvider.scala @@ -11,11 +11,10 @@ import common.exception._ import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.http.HttpStatus +class DrsCloudNioFileProvider(drsPathResolver: DrsPathResolver, drsReadInterpreter: DrsReadInterpreter) + extends CloudNioFileProvider { -class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver, - drsReadInterpreter: DrsReadInterpreter) extends CloudNioFileProvider { - - private def checkIfPathExistsThroughMartha(drsPath: String): IO[Boolean] = { + private def checkIfPathExistsThroughDrsResolver(drsPath: String): IO[Boolean] = /* * Unlike other cloud providers where directories are identified with a trailing slash at the end like `gs://bucket/dir/`, * DRS has a concept of bundles for directories (not supported yet). Hence for method `checkDirectoryExists` which appends a trailing '/' @@ -23,54 +22,68 @@ class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver, */ if (drsPath.endsWith("/")) IO(false) else { - drsPathResolver.rawMarthaResponse(drsPath, NonEmptyList.one(MarthaField.GsUri)).use { marthaResponse => - val errorMsg = s"Status line was null for martha response $marthaResponse." - toIO(Option(marthaResponse.getStatusLine), errorMsg) - }.map(_.getStatusCode == HttpStatus.SC_OK) + drsPathResolver + .rawDrsResolverResponse(drsPath, NonEmptyList.one(DrsResolverField.GsUri)) + .use { drsResolverResponse => + val errorMsg = s"Status line was null for DRS Resolver response $drsResolverResponse." + toIO(Option(drsResolverResponse.getStatusLine), errorMsg) + } + .map(_.getStatusCode == HttpStatus.SC_OK) } - } override def existsPath(drsPath: String, unused: String): Boolean = - checkIfPathExistsThroughMartha(drsPath).unsafeRunSync() + checkIfPathExistsThroughDrsResolver(drsPath).unsafeRunSync() override def existsPaths(cloudHost: String, cloudPathPrefix: String): Boolean = existsPath(cloudHost, cloudPathPrefix) - override def listObjects(drsPath: String, unused: String, markerOption: Option[String]): CloudNioFileList = { + override def listObjects(drsPath: String, unused: String, markerOption: Option[String]): CloudNioFileList = throw new UnsupportedOperationException("DRS currently doesn't support list.") - } - override def copy(sourceCloudHost: String, sourceCloudPath: String, targetCloudHost: String, targetCloudPath: String): Unit = + override def copy(sourceCloudHost: String, + sourceCloudPath: String, + targetCloudHost: String, + targetCloudPath: String + ): Unit = throw new UnsupportedOperationException("DRS currently doesn't support copy.") override def deleteIfExists(cloudHost: String, cloudPath: String): Boolean = throw new UnsupportedOperationException("DRS currently doesn't support delete.") override def read(drsPath: String, unused: String, offset: Long): ReadableByteChannel = { - val fields = NonEmptyList.of(MarthaField.GsUri, MarthaField.GoogleServiceAccount, MarthaField.AccessUrl) + val fields = + NonEmptyList.of(DrsResolverField.GsUri, DrsResolverField.GoogleServiceAccount, DrsResolverField.AccessUrl) val byteChannelIO = for { - marthaResponse <- drsPathResolver.resolveDrsThroughMartha(drsPath, fields) - byteChannel <- drsReadInterpreter(drsPathResolver, marthaResponse) + drsResolverResponse <- drsPathResolver.resolveDrs(drsPath, fields) + byteChannel <- drsReadInterpreter(drsPathResolver, drsResolverResponse) } yield byteChannel - byteChannelIO.handleErrorWith { - e => IO.raiseError(new RuntimeException(s"Error while reading from DRS path: $drsPath. Error: ${ExceptionUtils.getMessage(e)}")) - }.unsafeRunSync() + byteChannelIO + .handleErrorWith { e => + IO.raiseError( + new RuntimeException(s"Error while reading from DRS path: $drsPath. Error: ${ExceptionUtils.getMessage(e)}") + ) + } + .unsafeRunSync() } override def write(cloudHost: String, cloudPath: String): WritableByteChannel = throw new UnsupportedOperationException("DRS currently doesn't support write.") override def fileAttributes(drsPath: String, unused: String): Option[CloudNioRegularFileAttributes] = { - val fields = NonEmptyList.of(MarthaField.Size, MarthaField.TimeCreated, MarthaField.TimeUpdated, MarthaField.Hashes) + val fields = NonEmptyList.of(DrsResolverField.Size, + DrsResolverField.TimeCreated, + DrsResolverField.TimeUpdated, + DrsResolverField.Hashes + ) val fileAttributesIO = for { - marthaResponse <- drsPathResolver.resolveDrsThroughMartha(drsPath, fields) - sizeOption = marthaResponse.size - hashOption = getPreferredHash(marthaResponse.hashes) - timeCreatedOption <- convertToFileTime(drsPath, MarthaField.TimeCreated, marthaResponse.timeCreated) - timeUpdatedOption <- convertToFileTime(drsPath, MarthaField.TimeUpdated, marthaResponse.timeUpdated) + drsResolverResponse <- drsPathResolver.resolveDrs(drsPath, fields) + sizeOption = drsResolverResponse.size + hashOption = getPreferredHash(drsResolverResponse.hashes) + timeCreatedOption <- convertToFileTime(drsPath, DrsResolverField.TimeCreated, drsResolverResponse.timeCreated) + timeUpdatedOption <- convertToFileTime(drsPath, DrsResolverField.TimeUpdated, drsResolverResponse.timeUpdated) } yield new DrsCloudNioRegularFileAttributes(drsPath, sizeOption, hashOption, timeCreatedOption, timeUpdatedOption) Option(fileAttributesIO.unsafeRunSync()) @@ -78,5 +91,5 @@ class DrsCloudNioFileProvider(drsPathResolver: EngineDrsPathResolver, } object DrsCloudNioFileProvider { - type DrsReadInterpreter = (DrsPathResolver, MarthaResponse) => IO[ReadableByteChannel] + type DrsReadInterpreter = (DrsPathResolver, DrsResolverResponse) => IO[ReadableByteChannel] } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala index 8721ab9b43e..216c20524f6 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProvider.scala @@ -8,13 +8,13 @@ import com.typesafe.config.Config class DrsCloudNioFileSystemProvider(rootConfig: Config, val drsCredentials: DrsCredentials, - drsReadInterpreter: DrsReadInterpreter, - ) extends CloudNioFileSystemProvider { + drsReadInterpreter: DrsReadInterpreter +) extends CloudNioFileSystemProvider { - lazy val drsConfig: DrsConfig = DrsConfig.fromConfig(rootConfig.getConfig("martha")) + lazy val drsResolverConfig = rootConfig.getConfig("resolver") + lazy val drsConfig: DrsConfig = DrsConfig.fromConfig(drsResolverConfig) - lazy val drsPathResolver: EngineDrsPathResolver = - EngineDrsPathResolver(drsConfig, drsCredentials) + lazy val drsPathResolver: DrsPathResolver = new DrsPathResolver(drsConfig, drsCredentials) override def config: Config = rootConfig diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala index 3c9293f2bd7..b09606294c6 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCloudNioRegularFileAttributes.scala @@ -11,8 +11,8 @@ class DrsCloudNioRegularFileAttributes(drsPath: String, sizeOption: Option[Long], hashOption: Option[FileHash], timeCreatedOption: Option[FileTime], - timeUpdatedOption: Option[FileTime], - ) extends CloudNioRegularFileAttributes{ + timeUpdatedOption: Option[FileTime] +) extends CloudNioRegularFileAttributes { override def fileKey(): String = drsPath @@ -33,50 +33,47 @@ object DrsCloudNioRegularFileAttributes { ("etag", HashType.S3Etag) ) - def getPreferredHash(hashesOption: Option[Map[String, String]]): Option[FileHash] = { + def getPreferredHash(hashesOption: Option[Map[String, String]]): Option[FileHash] = hashesOption match { case Some(hashes: Map[String, String]) if hashes.nonEmpty => priorityHashList collectFirst { case (key, hashType) if hashes.contains(key) => FileHash(hashType, hashes(key)) } - // if no preferred hash was found, go ahead and return none because we don't support anything that the DRS object is offering + // if no preferred hash was found, go ahead and return none because we don't support anything that the DRS object is offering case _ => None } - } - private def convertToOffsetDateTime(timeInString: String): IO[OffsetDateTime] = { + private def convertToOffsetDateTime(timeInString: String): IO[OffsetDateTime] = // Here timeInString is assumed to be a ISO-8601 DateTime with timezone IO(OffsetDateTime.parse(timeInString)) - .handleErrorWith( - offsetDateTimeException => - // As a fallback timeInString is assumed to be a ISO-8601 DateTime without timezone - IO(LocalDateTime.parse(timeInString).atOffset(ZoneOffset.UTC)) - .handleErrorWith(_ => IO.raiseError(offsetDateTimeException)) + .handleErrorWith(offsetDateTimeException => + // As a fallback timeInString is assumed to be a ISO-8601 DateTime without timezone + IO(LocalDateTime.parse(timeInString).atOffset(ZoneOffset.UTC)) + .handleErrorWith(_ => IO.raiseError(offsetDateTimeException)) ) - } - private def convertToFileTime(timeInString: String): IO[FileTime] = { + private def convertToFileTime(timeInString: String): IO[FileTime] = convertToOffsetDateTime(timeInString) .map(_.toInstant) .map(FileTime.from) - } - def convertToFileTime(drsPath: String, key: MarthaField.Value, timeInStringOption: Option[String]): IO[Option[FileTime]] = { + def convertToFileTime(drsPath: String, + key: DrsResolverField.Value, + timeInStringOption: Option[String] + ): IO[Option[FileTime]] = timeInStringOption match { case None => IO.pure(None) case Some(timeInString) => convertToFileTime(timeInString) .map(Option(_)) - .handleErrorWith( - throwable => - IO.raiseError( - new RuntimeException( - s"Error while parsing '$key' value from Martha to FileTime for DRS path $drsPath. " + - s"Reason: ${ExceptionUtils.getMessage(throwable)}.", - throwable, - ) + .handleErrorWith(throwable => + IO.raiseError( + new RuntimeException( + s"Error while parsing '$key' value from DRS Resolver to FileTime for DRS path $drsPath. " + + s"Reason: ${ExceptionUtils.getMessage(throwable)}.", + throwable ) + ) ) } - } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala index e9dc760162e..28b1ee57b7a 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsConfig.scala @@ -6,61 +6,57 @@ import net.ceedubs.ficus.Ficus._ import scala.concurrent.duration._ import scala.language.postfixOps -final case class DrsConfig(marthaUrl: String, +final case class DrsConfig(drsResolverUrl: String, numRetries: Int, waitInitial: FiniteDuration, waitMaximum: FiniteDuration, waitMultiplier: Double, - waitRandomizationFactor: Double, - ) + waitRandomizationFactor: Double +) object DrsConfig { // If you update these values also update Filesystems.md! private val DefaultNumRetries = 3 - private val DefaultWaitInitial = 10 seconds - private val DefaultWaitMaximum = 30 seconds - private val DefaultWaitMultiplier = 1.5d + private val DefaultWaitInitial = 30 seconds + private val DefaultWaitMaximum = 60 seconds + private val DefaultWaitMultiplier = 1.25d private val DefaultWaitRandomizationFactor = 0.1 - private val EnvMarthaUrl = "MARTHA_URL" - private val EnvMarthaNumRetries = "MARTHA_NUM_RETRIES" - private val EnvMarthaWaitInitialSeconds = "MARTHA_WAIT_INITIAL_SECONDS" - private val EnvMarthaWaitMaximumSeconds = "MARTHA_WAIT_MAXIMUM_SECONDS" - private val EnvMarthaWaitMultiplier = "MARTHA_WAIT_MULTIPLIER" - private val EnvMarthaWaitRandomizationFactor = "MARTHA_WAIT_RANDOMIZATION_FACTOR" + private val EnvDrsResolverUrl = "DRS_RESOLVER_URL" + private val EnvDrsResolverNumRetries = "DRS_RESOLVER_NUM_RETRIES" + private val EnvDrsResolverWaitInitialSeconds = "DRS_RESOLVER_WAIT_INITIAL_SECONDS" + private val EnvDrsResolverWaitMaximumSeconds = "DRS_RESOLVER_WAIT_MAXIMUM_SECONDS" + private val EnvDrsResolverWaitMultiplier = "DRS_RESOLVER_WAIT_MULTIPLIER" + private val EnvDrsResolverWaitRandomizationFactor = "DRS_RESOLVER_WAIT_RANDOMIZATION_FACTOR" - def fromConfig(marthaConfig: Config): DrsConfig = { + def fromConfig(drsResolverConfig: Config): DrsConfig = DrsConfig( - marthaUrl = marthaConfig.getString("url"), - numRetries = marthaConfig.getOrElse("num-retries", DefaultNumRetries), - waitInitial = marthaConfig.getOrElse("wait-initial", DefaultWaitInitial), - waitMaximum = marthaConfig.getOrElse("wait-maximum", DefaultWaitMaximum), - waitMultiplier = marthaConfig.getOrElse("wait-multiplier", DefaultWaitMultiplier), - waitRandomizationFactor = - marthaConfig.getOrElse("wait-randomization-factor", DefaultWaitRandomizationFactor), + drsResolverUrl = drsResolverConfig.getString("url"), + numRetries = drsResolverConfig.getOrElse("num-retries", DefaultNumRetries), + waitInitial = drsResolverConfig.getOrElse("wait-initial", DefaultWaitInitial), + waitMaximum = drsResolverConfig.getOrElse("wait-maximum", DefaultWaitMaximum), + waitMultiplier = drsResolverConfig.getOrElse("wait-multiplier", DefaultWaitMultiplier), + waitRandomizationFactor = drsResolverConfig.getOrElse("wait-randomization-factor", DefaultWaitRandomizationFactor) ) - } - def fromEnv(env: Map[String, String]): DrsConfig = { + def fromEnv(env: Map[String, String]): DrsConfig = DrsConfig( - marthaUrl = env(EnvMarthaUrl), - numRetries = env.get(EnvMarthaNumRetries).map(_.toInt).getOrElse(DefaultNumRetries), - waitInitial = env.get(EnvMarthaWaitInitialSeconds).map(_.toLong.seconds).getOrElse(DefaultWaitInitial), - waitMaximum = env.get(EnvMarthaWaitMaximumSeconds).map(_.toLong.seconds).getOrElse(DefaultWaitMaximum), - waitMultiplier = env.get(EnvMarthaWaitMultiplier).map(_.toDouble).getOrElse(DefaultWaitMultiplier), + drsResolverUrl = env(EnvDrsResolverUrl), + numRetries = env.get(EnvDrsResolverNumRetries).map(_.toInt).getOrElse(DefaultNumRetries), + waitInitial = env.get(EnvDrsResolverWaitInitialSeconds).map(_.toLong.seconds).getOrElse(DefaultWaitInitial), + waitMaximum = env.get(EnvDrsResolverWaitMaximumSeconds).map(_.toLong.seconds).getOrElse(DefaultWaitMaximum), + waitMultiplier = env.get(EnvDrsResolverWaitMultiplier).map(_.toDouble).getOrElse(DefaultWaitMultiplier), waitRandomizationFactor = - env.get(EnvMarthaWaitRandomizationFactor).map(_.toDouble).getOrElse(DefaultWaitRandomizationFactor), + env.get(EnvDrsResolverWaitRandomizationFactor).map(_.toDouble).getOrElse(DefaultWaitRandomizationFactor) ) - } - def toEnv(drsConfig: DrsConfig): Map[String, String] = { + def toEnv(drsConfig: DrsConfig): Map[String, String] = Map( - EnvMarthaUrl -> drsConfig.marthaUrl, - EnvMarthaNumRetries -> s"${drsConfig.numRetries}", - EnvMarthaWaitInitialSeconds -> s"${drsConfig.waitInitial.toSeconds}", - EnvMarthaWaitMaximumSeconds -> s"${drsConfig.waitMaximum.toSeconds}", - EnvMarthaWaitMultiplier -> s"${drsConfig.waitMultiplier}", - EnvMarthaWaitRandomizationFactor -> s"${drsConfig.waitRandomizationFactor}", + EnvDrsResolverUrl -> drsConfig.drsResolverUrl, + EnvDrsResolverNumRetries -> s"${drsConfig.numRetries}", + EnvDrsResolverWaitInitialSeconds -> s"${drsConfig.waitInitial.toSeconds}", + EnvDrsResolverWaitMaximumSeconds -> s"${drsConfig.waitMaximum.toSeconds}", + EnvDrsResolverWaitMultiplier -> s"${drsConfig.waitMultiplier}", + EnvDrsResolverWaitRandomizationFactor -> s"${drsConfig.waitRandomizationFactor}" ) - } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala index 66a89fe4014..c8a4fb93743 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsCredentials.scala @@ -1,30 +1,32 @@ package cloud.nio.impl.drs import cats.syntax.validated._ -import com.azure.identity.DefaultAzureCredentialBuilder -import com.azure.security.keyvault.secrets.{SecretClient, SecretClientBuilder} -import com.google.auth.oauth2.{AccessToken, OAuth2Credentials} +import com.google.auth.oauth2.{AccessToken, GoogleCredentials, OAuth2Credentials} import com.typesafe.config.Config -import common.validation.ErrorOr import common.validation.ErrorOr.ErrorOr import net.ceedubs.ficus.Ficus._ +import cromwell.cloudsupport.azure.AzureCredentials import scala.concurrent.duration._ +import scala.util.{Failure, Success, Try} /** * This trait allows us to abstract away different token attainment strategies * for different cloud environments. **/ -sealed trait DrsCredentials { +trait DrsCredentials { def getAccessToken: ErrorOr[String] } -case class GoogleDrsCredentials(credentials: OAuth2Credentials, acceptableTTL: Duration) extends DrsCredentials { - //Based on method from GoogleRegistry +/** + * Strategy for obtaining an access token from an existing OAuth credential. This class + * is designed for use within the Cromwell engine. + */ +case class GoogleOauthDrsCredentials(credentials: OAuth2Credentials, acceptableTTL: Duration) extends DrsCredentials { + // Based on method from GoogleRegistry def getAccessToken: ErrorOr[String] = { - def accessTokenTTLIsAcceptable(accessToken: AccessToken): Boolean = { + def accessTokenTTLIsAcceptable(accessToken: AccessToken): Boolean = (accessToken.getExpirationTime.getTime - System.currentTimeMillis()).millis.gteq(acceptableTTL) - } Option(credentials.getAccessToken) match { case Some(accessToken) if accessTokenTTLIsAcceptable(accessToken) => @@ -39,25 +41,37 @@ case class GoogleDrsCredentials(credentials: OAuth2Credentials, acceptableTTL: D } } -object GoogleDrsCredentials { - def apply(credentials: OAuth2Credentials, config: Config): GoogleDrsCredentials = - GoogleDrsCredentials(credentials, config.as[FiniteDuration]("access-token-acceptable-ttl")) +object GoogleOauthDrsCredentials { + def apply(credentials: OAuth2Credentials, config: Config): GoogleOauthDrsCredentials = + GoogleOauthDrsCredentials(credentials, config.as[FiniteDuration]("access-token-acceptable-ttl")) } -case class AzureDrsCredentials(identityClientId: Option[String], vaultName: String, secretName: String) extends DrsCredentials { - - lazy val secretClient: ErrorOr[SecretClient] = ErrorOr { - val defaultCreds = identityClientId.map(identityId => - new DefaultAzureCredentialBuilder().managedIdentityClientId(identityId) - ).getOrElse( - new DefaultAzureCredentialBuilder() - ).build() +/** + * Strategy for obtaining an access token from Google Application Default credentials that are assumed to already exist + * in the environment. This class is designed for use by standalone executables running in environments + * that have direct access to a Google identity (ex. CromwellDrsLocalizer). + */ +case object GoogleAppDefaultTokenStrategy extends DrsCredentials { + final private val UserInfoEmailScope = "https://www.googleapis.com/auth/userinfo.email" + final private val UserInfoProfileScope = "https://www.googleapis.com/auth/userinfo.profile" - new SecretClientBuilder() - .vaultUrl(s"https://${vaultName}.vault.azure.net") - .credential(defaultCreds) - .buildClient() - } + def getAccessToken: ErrorOr[String] = + Try { + val scopedCredentials = + GoogleCredentials.getApplicationDefault().createScoped(UserInfoEmailScope, UserInfoProfileScope) + scopedCredentials.refreshAccessToken().getTokenValue + } match { + case Success(null) => "null token value attempting to refresh access token".invalidNel + case Success(value) => value.validNel + case Failure(e) => s"Failed to refresh access token: ${e.getMessage}".invalidNel + } +} - def getAccessToken: ErrorOr[String] = secretClient.map(_.getSecret(secretName).getValue) +/** + * Strategy for obtaining an access token in an environment with available Azure identity. + * If you need to disambiguate among multiple active user-assigned managed identities, pass + * in the client id of the identity that should be used. + */ +case class AzureDrsCredentials(identityClientId: Option[String] = None) extends DrsCredentials { + def getAccessToken: ErrorOr[String] = AzureCredentials.getAccessToken(identityClientId) } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala index 9e0b45e259e..028d3ee4849 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsPathResolver.scala @@ -5,8 +5,8 @@ import cats.data.Validated.{Invalid, Valid} import cats.effect.{IO, Resource} import cats.implicits._ import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDisposition} -import cloud.nio.impl.drs.MarthaResponseSupport._ -import common.exception.{AggregatedMessageException, toIO} +import cloud.nio.impl.drs.DrsResolverResponseSupport._ +import common.exception.{toIO, AggregatedMessageException} import common.validation.ErrorOr.ErrorOr import io.circe._ import io.circe.generic.semiauto._ @@ -17,6 +17,7 @@ import org.apache.commons.lang3.exception.ExceptionUtils import org.apache.http.client.methods.{HttpGet, HttpPost} import org.apache.http.entity.{ContentType, StringEntity} import org.apache.http.impl.client.HttpClientBuilder +import org.apache.http.impl.conn.PoolingHttpClientConnectionManager import org.apache.http.util.EntityUtils import org.apache.http.{HttpResponse, HttpStatus, StatusLine} @@ -24,37 +25,52 @@ import java.nio.ByteBuffer import java.nio.channels.{Channels, ReadableByteChannel} import scala.util.Try -abstract class DrsPathResolver(drsConfig: DrsConfig, retryInternally: Boolean = true) { +class DrsPathResolver(drsConfig: DrsConfig, drsCredentials: DrsCredentials) { protected lazy val httpClientBuilder: HttpClientBuilder = { val clientBuilder = HttpClientBuilder.create() - if (retryInternally) { - val retryHandler = new MarthaHttpRequestRetryStrategy(drsConfig) - clientBuilder - .setRetryHandler(retryHandler) - .setServiceUnavailableRetryStrategy(retryHandler) - } + val retryHandler = new DrsResolverHttpRequestRetryStrategy(drsConfig) + clientBuilder + .setRetryHandler(retryHandler) + .setServiceUnavailableRetryStrategy(retryHandler) + clientBuilder.setConnectionManager(connectionManager) + clientBuilder.setConnectionManagerShared(true) clientBuilder } - def getAccessToken: ErrorOr[String] + def getAccessToken: ErrorOr[String] = drsCredentials.getAccessToken + + private lazy val currentCloudPlatform: Option[DrsCloudPlatform.Value] = drsCredentials match { + case _: GoogleOauthDrsCredentials => Option(DrsCloudPlatform.GoogleStorage) + case GoogleAppDefaultTokenStrategy => Option(DrsCloudPlatform.GoogleStorage) + case _: AzureDrsCredentials => Option(DrsCloudPlatform.Azure) + case _ => None + } + + def makeDrsResolverRequest(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): DrsResolverRequest = + DrsResolverRequest(drsPath, currentCloudPlatform, fields) - private def makeHttpRequestToMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): Resource[IO, HttpPost] = { + private def makeHttpRequestToDrsResolver(drsPath: String, + fields: NonEmptyList[DrsResolverField.Value] + ): Resource[IO, HttpPost] = { val io = getAccessToken match { - case Valid(token) => IO { - val postRequest = new HttpPost(drsConfig.marthaUrl) - val requestJson = MarthaRequest(drsPath, fields).asJson.noSpaces - postRequest.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) - postRequest.setHeader("Authorization", s"Bearer $token") - postRequest - } + case Valid(token) => + IO { + val postRequest = new HttpPost(drsConfig.drsResolverUrl) + val requestJson = makeDrsResolverRequest(drsPath, fields).asJson.noSpaces + postRequest.setEntity(new StringEntity(requestJson, ContentType.APPLICATION_JSON)) + postRequest.setHeader("Authorization", s"Bearer $token") + postRequest + } case Invalid(errors) => IO.raiseError(AggregatedMessageException("Error getting access token", errors.toList)) } Resource.eval(io) } - private def httpResponseToMarthaResponse(drsPathForDebugging: String)(httpResponse: HttpResponse): IO[MarthaResponse] = { + private def httpResponseToDrsResolverResponse( + drsPathForDebugging: String + )(httpResponse: HttpResponse): IO[DrsResolverResponse] = { val responseStatusLine = httpResponse.getStatusLine val status = responseStatusLine.getStatusCode @@ -71,46 +87,56 @@ abstract class DrsPathResolver(drsConfig: DrsConfig, retryInternally: Boolean = case s if s / 100 == 5 => IO.raiseError(new RuntimeException(retryMessage) with RegularRetryDisposition) case _ => - val marthaResponseEntityOption = Option(httpResponse.getEntity).map(EntityUtils.toString) - val exceptionMsg = errorMessageFromResponse(drsPathForDebugging, marthaResponseEntityOption, responseStatusLine, drsConfig.marthaUrl) - val responseEntityOption = (responseStatusLine.getStatusCode == HttpStatus.SC_OK).valueOrZero(marthaResponseEntityOption) + val drsResolverResponseEntityOption = Option(httpResponse.getEntity).map(EntityUtils.toString) + val exceptionMsg = errorMessageFromResponse(drsPathForDebugging, + drsResolverResponseEntityOption, + responseStatusLine, + drsConfig.drsResolverUrl + ) + val responseEntityOption = + (responseStatusLine.getStatusCode == HttpStatus.SC_OK).valueOrZero(drsResolverResponseEntityOption) val responseContentIO = toIO(responseEntityOption, exceptionMsg) - responseContentIO.flatMap { responseContent => - IO.fromEither(decode[MarthaResponse](responseContent)) - }.handleErrorWith { - e => IO.raiseError(new RuntimeException(s"Unexpected response during DRS resolution: ${ExceptionUtils.getMessage(e)}")) - } + responseContentIO + .flatMap { responseContent => + IO.fromEither(decode[DrsResolverResponse](responseContent)) + } + .handleErrorWith { e => + IO.raiseError( + new RuntimeException(s"Unexpected response during DRS resolution: ${ExceptionUtils.getMessage(e)}") + ) + } } } - private def executeMarthaRequest(httpPost: HttpPost): Resource[IO, HttpResponse]= { + private def executeDrsResolverRequest(httpPost: HttpPost): Resource[IO, HttpResponse] = for { httpClient <- Resource.fromAutoCloseable(IO(httpClientBuilder.build())) httpResponse <- Resource.fromAutoCloseable(IO(httpClient.execute(httpPost))) } yield httpResponse - } - def rawMarthaResponse(drsPath: String, fields: NonEmptyList[MarthaField.Value]): Resource[IO, HttpResponse] = { + def rawDrsResolverResponse(drsPath: String, + fields: NonEmptyList[DrsResolverField.Value] + ): Resource[IO, HttpResponse] = for { - httpPost <- makeHttpRequestToMartha(drsPath, fields) - response <- executeMarthaRequest(httpPost) + httpPost <- makeHttpRequestToDrsResolver(drsPath, fields) + response <- executeDrsResolverRequest(httpPost) } yield response - } /** * - * Resolves the DRS path through Martha url provided in the config. - * Please note, this method returns an IO that would make a synchronous HTTP request to Martha when run. + * Resolves the DRS path through DRS Resolver url provided in the config. + * Please note, this method returns an IO that would make a synchronous HTTP request to DRS Resolver when run. */ - def resolveDrsThroughMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): IO[MarthaResponse] = { - rawMarthaResponse(drsPath, fields).use(httpResponseToMarthaResponse(drsPathForDebugging = drsPath)) - } + def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = + rawDrsResolverResponse(drsPath, fields).use( + httpResponseToDrsResolverResponse(drsPathForDebugging = drsPath) + ) - def openChannel(accessUrl: AccessUrl): IO[ReadableByteChannel] = { + def openChannel(accessUrl: AccessUrl): IO[ReadableByteChannel] = IO { val httpGet = new HttpGet(accessUrl.url) - accessUrl.headers.getOrElse(Map.empty).toList foreach { - case (name, value) => httpGet.addHeader(name, value) + accessUrl.headers.getOrElse(Map.empty).toList foreach { case (name, value) => + httpGet.addHeader(name, value) } val client = httpClientBuilder.build() val response = client.execute(httpGet) @@ -129,7 +155,7 @@ abstract class DrsPathResolver(drsConfig: DrsConfig, retryInternally: Boolean = override def isOpen: Boolean = inner.isOpen - //noinspection ScalaUnusedExpression + // noinspection ScalaUnusedExpression override def close(): Unit = { val innerTry = Try(inner.close()) val responseTry = Try(response.close()) @@ -140,11 +166,10 @@ abstract class DrsPathResolver(drsConfig: DrsConfig, retryInternally: Boolean = } } } - } } object DrsPathResolver { - final val ExtractUriErrorMsg = "No access URL nor GCS URI starting with 'gs://' found in Martha response!" + final val ExtractUriErrorMsg = "No access URL nor GCS URI starting with 'gs://' found in the DRS Resolver response!" sealed trait RetryDisposition // Should immediately fail the download attempt. trait FatalRetryDisposition extends RetryDisposition @@ -152,27 +177,40 @@ object DrsPathResolver { trait RegularRetryDisposition extends RetryDisposition } -object MarthaField extends Enumeration { - val GsUri: MarthaField.Value = Value("gsUri") - val Size: MarthaField.Value = Value("size") - val TimeCreated: MarthaField.Value = Value("timeCreated") - val TimeUpdated: MarthaField.Value = Value("timeUpdated") - val BondProvider: MarthaField.Value = Value("bondProvider") - val GoogleServiceAccount: MarthaField.Value = Value("googleServiceAccount") - val Hashes: MarthaField.Value = Value("hashes") - val FileName: MarthaField.Value = Value("fileName") - val AccessUrl: MarthaField.Value = Value("accessUrl") - val LocalizationPath: MarthaField.Value = Value("localizationPath") +object DrsResolverField extends Enumeration { + val GsUri: DrsResolverField.Value = Value("gsUri") + val Size: DrsResolverField.Value = Value("size") + val TimeCreated: DrsResolverField.Value = Value("timeCreated") + val TimeUpdated: DrsResolverField.Value = Value("timeUpdated") + val BondProvider: DrsResolverField.Value = Value("bondProvider") + val GoogleServiceAccount: DrsResolverField.Value = Value("googleServiceAccount") + val Hashes: DrsResolverField.Value = Value("hashes") + val FileName: DrsResolverField.Value = Value("fileName") + val AccessUrl: DrsResolverField.Value = Value("accessUrl") + val LocalizationPath: DrsResolverField.Value = Value("localizationPath") } -final case class MarthaRequest(url: String, fields: NonEmptyList[MarthaField.Value]) +// We supply a cloud platform value to the DRS service. In cases where the DRS repository +// has multiple cloud files associated with a DRS link, it will prefer sending a file on the same +// platform as this Cromwell instance. That is, if a DRS file has copies on both GCP and Azure, +// we'll get the GCP one when running on GCP and the Azure one when running on Azure. +object DrsCloudPlatform extends Enumeration { + val GoogleStorage: DrsCloudPlatform.Value = Value("gs") + val Azure: DrsCloudPlatform.Value = Value("azure") + val AmazonS3: DrsCloudPlatform.Value = Value("s3") // supported by DRSHub but not currently used by us +} + +final case class DrsResolverRequest(url: String, + cloudPlatform: Option[DrsCloudPlatform.Value], + fields: NonEmptyList[DrsResolverField.Value] +) final case class SADataObject(data: Json) final case class AccessUrl(url: String, headers: Option[Map[String, String]]) /** - * A response from `martha_v3`. + * A response from `drshub_v4`. * * @param size Size of the object stored at gsUri * @param timeCreated The creation time of the object at gsUri @@ -187,54 +225,61 @@ final case class AccessUrl(url: String, headers: Option[Map[String, String]]) * DRS metadata, via the `aliases` field. As this is a distinct field from `fileName` in DRS * metadata it is also made a distinct field in this response object. */ -final case class MarthaResponse(size: Option[Long] = None, - timeCreated: Option[String] = None, - timeUpdated: Option[String] = None, - gsUri: Option[String] = None, - bondProvider: Option[String] = None, - googleServiceAccount: Option[SADataObject] = None, - fileName: Option[String] = None, - hashes: Option[Map[String, String]] = None, - accessUrl: Option[AccessUrl] = None, - localizationPath: Option[String] = None - ) - -// Adapted from https://github.com/broadinstitute/martha/blob/f31933a3a11e20d30698ec4b4dc1e0abbb31a8bc/common/helpers.js#L210-L218 -final case class MarthaFailureResponse(response: MarthaFailureResponsePayload) -final case class MarthaFailureResponsePayload(text: String) - -object MarthaResponseSupport { - - implicit lazy val marthaFieldEncoder: Encoder[MarthaField.Value] = Encoder.encodeEnumeration(MarthaField) - implicit lazy val marthaRequestEncoder: Encoder[MarthaRequest] = deriveEncoder +final case class DrsResolverResponse(size: Option[Long] = None, + timeCreated: Option[String] = None, + timeUpdated: Option[String] = None, + gsUri: Option[String] = None, + bondProvider: Option[String] = None, + googleServiceAccount: Option[SADataObject] = None, + fileName: Option[String] = None, + hashes: Option[Map[String, String]] = None, + accessUrl: Option[AccessUrl] = None, + localizationPath: Option[String] = None +) + +final case class DrsResolverFailureResponse(response: DrsResolverFailureResponsePayload) +final case class DrsResolverFailureResponsePayload(text: String) + +object DrsResolverResponseSupport { + + implicit lazy val drsResolverFieldEncoder: Encoder[DrsResolverField.Value] = + Encoder.encodeEnumeration(DrsResolverField) + implicit lazy val drsResolverCloudPlatformEncoder: Encoder[DrsCloudPlatform.Value] = + Encoder.encodeEnumeration(DrsCloudPlatform) + implicit lazy val drsResolverRequestEncoder: Encoder[DrsResolverRequest] = deriveEncoder implicit lazy val saDataObjectDecoder: Decoder[SADataObject] = deriveDecoder - implicit lazy val marthaResponseDecoder: Decoder[MarthaResponse] = deriveDecoder + implicit lazy val drsResolverResponseDecoder: Decoder[DrsResolverResponse] = deriveDecoder - implicit lazy val marthaFailureResponseDecoder: Decoder[MarthaFailureResponse] = deriveDecoder - implicit lazy val marthaFailureResponsePayloadDecoder: Decoder[MarthaFailureResponsePayload] = deriveDecoder + implicit lazy val drsResolverFailureResponseDecoder: Decoder[DrsResolverFailureResponse] = deriveDecoder + implicit lazy val drsResolverFailureResponsePayloadDecoder: Decoder[DrsResolverFailureResponsePayload] = deriveDecoder - implicit lazy val marthaAccessUrlDecoder: Decoder[AccessUrl] = deriveDecoder + implicit lazy val drsResolverAccessUrlDecoder: Decoder[AccessUrl] = deriveDecoder private val GcsScheme = "gs://" def getGcsBucketAndName(gcsUrl: String): (String, String) = { - val array = gcsUrl.substring(GcsScheme.length).split("/", 2) - (array(0), array(1)) + val array = gcsUrl.substring(GcsScheme.length).split("/", 2) + (array(0), array(1)) } - def errorMessageFromResponse(drsPathForDebugging: String, marthaResponseEntityOption: Option[String], responseStatusLine: StatusLine, marthaUri: String): String = { - val baseMessage = s"Could not access object \'$drsPathForDebugging\'. Status: ${responseStatusLine.getStatusCode}, reason: \'${responseStatusLine.getReasonPhrase}\', Martha location: \'$marthaUri\', message: " + def errorMessageFromResponse(drsPathForDebugging: String, + drsResolverResponseEntityOption: Option[String], + responseStatusLine: StatusLine, + drsResolverUri: String + ): String = { + val baseMessage = + s"Could not access object \'$drsPathForDebugging\'. Status: ${responseStatusLine.getStatusCode}, reason: \'${responseStatusLine.getReasonPhrase}\', DRS Resolver location: \'$drsResolverUri\', message: " - marthaResponseEntityOption match { + drsResolverResponseEntityOption match { case Some(entity) => - val maybeErrorResponse: Either[Error, MarthaFailureResponse] = decode[MarthaFailureResponse](entity) + val maybeErrorResponse: Either[Error, DrsResolverFailureResponse] = decode[DrsResolverFailureResponse](entity) maybeErrorResponse match { case Left(_) => - // Not parsable as a `MarthaFailureResponse` + // Not parsable as a `DrsResolverFailureResponse` baseMessage + s"\'$entity\'" case Right(decoded) => - // Is a `MarthaFailureResponse` + // Is a `DrsResolverFailureResponse` baseMessage + s"\'${decoded.response.text}\'" } case None => @@ -242,4 +287,13 @@ object MarthaResponseSupport { baseMessage + "(empty response)" } } + + lazy val connectionManager = { + val connManager = new PoolingHttpClientConnectionManager() + connManager.setMaxTotal(250) + // Since the HttpClient is always talking to DRSHub, + // make the max connections per route the same as max total connections + connManager.setDefaultMaxPerRoute(250) + connManager + } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/MarthaHttpRequestRetryStrategy.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategy.scala similarity index 82% rename from cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/MarthaHttpRequestRetryStrategy.scala rename to cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategy.scala index 2881fc60b05..e39e7b49c81 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/MarthaHttpRequestRetryStrategy.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategy.scala @@ -7,8 +7,9 @@ import org.apache.http.HttpResponse import org.apache.http.client.{HttpRequestRetryHandler, ServiceUnavailableRetryStrategy} import org.apache.http.protocol.HttpContext -class MarthaHttpRequestRetryStrategy(drsConfig: DrsConfig) - extends ServiceUnavailableRetryStrategy with HttpRequestRetryHandler { +class DrsResolverHttpRequestRetryStrategy(drsConfig: DrsConfig) + extends ServiceUnavailableRetryStrategy + with HttpRequestRetryHandler { // We can execute a total of one time, plus the number of retries private val executionMax: Int = drsConfig.numRetries + 1 @@ -17,23 +18,21 @@ class MarthaHttpRequestRetryStrategy(drsConfig: DrsConfig) initialInterval = drsConfig.waitInitial, maxInterval = drsConfig.waitMaximum, multiplier = drsConfig.waitMultiplier, - randomizationFactor = drsConfig.waitRandomizationFactor, + randomizationFactor = drsConfig.waitRandomizationFactor ) private var transientFailures: Int = 0 /** Returns true if an IOException should be immediately retried. */ - override def retryRequest(exception: IOException, executionCount: Int, context: HttpContext): Boolean = { + override def retryRequest(exception: IOException, executionCount: Int, context: HttpContext): Boolean = retryRequest(executionCount) - } /** Returns true if HttpResponse should be retried after getRetryInterval. */ - override def retryRequest(response: HttpResponse, executionCount: Int, context: HttpContext): Boolean = { + override def retryRequest(response: HttpResponse, executionCount: Int, context: HttpContext): Boolean = response.getStatusLine.getStatusCode match { case code if code == 408 || code == 429 => retryRequestTransient(executionCount) case code if 500 <= code && code <= 599 => retryRequest(executionCount) case _ => false } - } /** Returns the number of milliseconds to wait before retrying an HttpResponse. */ override def getRetryInterval: Long = { @@ -47,8 +46,7 @@ class MarthaHttpRequestRetryStrategy(drsConfig: DrsConfig) retryRequest(executionCount) } - private def retryRequest(executionCount: Int): Boolean = { + private def retryRequest(executionCount: Int): Boolean = // The first execution is executionCount == 1 executionCount - transientFailures <= executionMax - } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala deleted file mode 100644 index a62ce7971c2..00000000000 --- a/cloud-nio/cloud-nio-impl-drs/src/main/scala/cloud/nio/impl/drs/EngineDrsPathResolver.scala +++ /dev/null @@ -1,11 +0,0 @@ -package cloud.nio.impl.drs - -import common.validation.ErrorOr.ErrorOr - -case class EngineDrsPathResolver(drsConfig: DrsConfig, - drsCredentials: DrsCredentials, - ) - extends DrsPathResolver(drsConfig, retryInternally = false) { - - override def getAccessToken: ErrorOr[String] = drsCredentials.getAccessToken -} diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala index fd7b71ef4ee..c3b7cb70349 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileProviderSpec.scala @@ -25,15 +25,15 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp it should "parse a config and create a working file system provider" in { val config = ConfigFactory.parseString( - """martha.url = "https://from.config" + """resolver.url = "https://from.config" |access-token-acceptable-ttl = 1 minute |""".stripMargin ) val fileSystemProvider = new MockDrsCloudNioFileSystemProvider(config = config) - fileSystemProvider.drsConfig.marthaUrl should be("https://from.config") + fileSystemProvider.drsConfig.drsResolverUrl should be("https://from.config") fileSystemProvider.drsCredentials match { - case GoogleDrsCredentials(_, ttl) => ttl should be(1.minute) + case GoogleOauthDrsCredentials(_, ttl) => ttl should be(1.minute) case error => fail(s"Expected GoogleDrsCredentials, found $error") } fileSystemProvider.fileProvider should be(a[DrsCloudNioFileProvider]) @@ -52,11 +52,10 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp "drs://dg.4DFC:0027045b-9ed6-45af-a68e-f55037b5184c", "drs://dg.4503:dg.4503/fc046e84-6cf9-43a3-99cc-ffa2964b88cb", "drs://dg.ANV0:dg.ANV0/0db6577e-57bd-48a1-93c6-327c292bcb6b", - "drs://dg.F82A1A:ed6be7ab-068e-46c8-824a-f39cfbb885cc", + "drs://dg.F82A1A:ed6be7ab-068e-46c8-824a-f39cfbb885cc" ) - for (exampleUri <- exampleUris) { + for (exampleUri <- exampleUris) fileSystemProvider.getHost(exampleUri) should be(exampleUri) - } } it should "check existing drs objects" in { @@ -78,70 +77,62 @@ class DrsCloudNioFileProviderSpec extends AnyFlatSpecLike with CromwellTimeoutSp } it should "return a file provider that can read bytes from gcs" in { - val drsPathResolver = new MockEngineDrsPathResolver() { - override def resolveDrsThroughMartha(drsPath: String, - fields: NonEmptyList[MarthaField.Value], - ): IO[MarthaResponse] = { - IO(MarthaResponse(gsUri = Option("gs://bucket/object/path"))) - } + val drsPathResolver = new MockDrsPathResolver() { + override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = + IO(DrsResolverResponse(gsUri = Option("gs://bucket/object/path"))) } val readChannel = mock[ReadableByteChannel] - val drsReadInterpreter: DrsReadInterpreter = (_, marthaResponse) => { + val drsReadInterpreter: DrsReadInterpreter = (_, drsResolverResponse) => IO( - (marthaResponse.gsUri, marthaResponse.googleServiceAccount) match { + (drsResolverResponse.gsUri, drsResolverResponse.googleServiceAccount) match { case (Some("gs://bucket/object/path"), None) => readChannel - case _ => fail(s"Unexpected parameters passed: $marthaResponse") + case _ => fail(s"Unexpected parameters passed: $drsResolverResponse") } ) - } val fileSystemProvider = new MockDrsCloudNioFileSystemProvider( mockResolver = Option(drsPathResolver), - drsReadInterpreter = drsReadInterpreter, + drsReadInterpreter = drsReadInterpreter ) fileSystemProvider.fileProvider.read("dg.123", "abc", 0) should be(readChannel) } it should "return a file provider that can read bytes from an access url" in { - val drsPathResolver = new MockEngineDrsPathResolver() { - override def resolveDrsThroughMartha(drsPath: String, - fields: NonEmptyList[MarthaField.Value], - ): IO[MarthaResponse] = { - IO(MarthaResponse(accessUrl = Option(AccessUrl("https://host/object/path", None)))) - } + val drsPathResolver = new MockDrsPathResolver() { + override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = + IO(DrsResolverResponse(accessUrl = Option(AccessUrl("https://host/object/path", None)))) } val readChannel = mock[ReadableByteChannel] - val drsReadInterpreter: DrsReadInterpreter = (_, marthaResponse) => { + val drsReadInterpreter: DrsReadInterpreter = (_, drsResolverResponse) => IO( - marthaResponse.accessUrl match { + drsResolverResponse.accessUrl match { case Some(AccessUrl("https://host/object/path", None)) => readChannel - case _ => fail(s"Unexpected parameters passed: $marthaResponse") + case _ => fail(s"Unexpected parameters passed: $drsResolverResponse") } ) - } val fileSystemProvider = new MockDrsCloudNioFileSystemProvider( mockResolver = Option(drsPathResolver), - drsReadInterpreter = drsReadInterpreter, + drsReadInterpreter = drsReadInterpreter ) fileSystemProvider.fileProvider.read("dg.123", "abc", 0) should be(readChannel) } it should "return a file provider that can return file attributes" in { - val drsPathResolver = new MockEngineDrsPathResolver() { - override def resolveDrsThroughMartha(drsPath: String, - fields: NonEmptyList[MarthaField.Value], - ): IO[MarthaResponse] = { + val drsPathResolver = new MockDrsPathResolver() { + override def resolveDrs(drsPath: String, + fields: NonEmptyList[DrsResolverField.Value] + ): IO[DrsResolverResponse] = { val instantCreated = Instant.ofEpochMilli(123L) val instantUpdated = Instant.ofEpochMilli(456L) IO( - MarthaResponse( + DrsResolverResponse( size = Option(789L), timeCreated = Option(OffsetDateTime.ofInstant(instantCreated, ZoneOffset.UTC).toString), timeUpdated = Option(OffsetDateTime.ofInstant(instantUpdated, ZoneOffset.UTC).toString), - hashes = Option(Map("md5" -> "gg0217869")), + hashes = Option(Map("md5" -> "gg0217869")) ) ) } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala index 88b95dc76e8..13df061d7a6 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsCloudNioFileSystemProviderSpec.scala @@ -17,6 +17,6 @@ class DrsCloudNioFileSystemProviderSpec extends org.scalatest.flatspec.AnyFlatSp val path = fileSystemProvider.getCloudNioPath("drs://foo/bar/") the[UnsupportedOperationException] thrownBy { fileSystemProvider.deleteIfExists(path) - } should have message("DRS currently doesn't support delete.") + } should have message "DRS currently doesn't support delete." } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala index 78585c96ed9..adbb2adf43b 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsPathResolverSpec.scala @@ -1,8 +1,9 @@ package cloud.nio.impl.drs +import cats.data.NonEmptyList + import java.nio.file.attribute.FileTime import java.time.OffsetDateTime - import cloud.nio.impl.drs.DrsCloudNioRegularFileAttributes._ import cloud.nio.spi.{FileHash, HashType} import common.assertion.CromwellTimeoutSpec @@ -12,12 +13,12 @@ import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matchers { - private val mockGSA = SADataObject(data = Json.fromJsonObject(JsonObject("key"-> Json.fromString("value")))) + private val mockGSA = SADataObject(data = Json.fromJsonObject(JsonObject("key" -> Json.fromString("value")))) private val crcHashValue = "8a366443" private val md5HashValue = "336ea55913bc261b72875bd259753046" private val shaHashValue = "f76877f8e86ec3932fd2ae04239fbabb8c90199dab0019ae55fa42b31c314c44" - private val fullMarthaResponse = MarthaResponse( + private val fullDrsResolverResponse = DrsResolverResponse( size = Option(34905345), timeCreated = Option(OffsetDateTime.parse("2020-04-27T15:56:09.696Z").toString), timeUpdated = Option(OffsetDateTime.parse("2020-04-27T15:56:09.696Z").toString), @@ -27,67 +28,83 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with hashes = Option(Map("md5" -> md5HashValue, "crc32c" -> crcHashValue)) ) - private val fullMarthaResponseNoTz = - fullMarthaResponse - .copy(timeUpdated = fullMarthaResponse.timeUpdated.map(_.stripSuffix("Z"))) + private val fullDrsResolverResponseNoTz = + fullDrsResolverResponse + .copy(timeUpdated = fullDrsResolverResponse.timeUpdated.map(_.stripSuffix("Z"))) - private val fullMarthaResponseNoTime = - fullMarthaResponse + private val fullDrsResolverResponseNoTime = + fullDrsResolverResponse .copy(timeUpdated = None) - private val fullMarthaResponseBadTz = - fullMarthaResponse - .copy(timeUpdated = fullMarthaResponse.timeUpdated.map(_.stripSuffix("Z") + "BADTZ")) + private val fullDrsResolverResponseBadTz = + fullDrsResolverResponse + .copy(timeUpdated = fullDrsResolverResponse.timeUpdated.map(_.stripSuffix("Z") + "BADTZ")) private val etagHashValue = "something" - private val completeHashesMap = Option(Map( - "betty" -> "abc123", - "charles" -> "456", - "alfred" -> "xrd", - "sha256" -> shaHashValue, - "crc32c" -> crcHashValue, - "md5" -> md5HashValue, - "etag" -> etagHashValue, - )) - - private val missingCRCHashesMap = Option(Map( - "alfred" -> "xrd", - "sha256" -> shaHashValue, - "betty" -> "abc123", - "md5" -> md5HashValue, - "charles" -> "456", - )) - - private val onlySHAHashesMap = Option(Map( - "betty" -> "abc123", - "charles" -> "456", - "alfred" -> "xrd", - "sha256" -> shaHashValue, - )) - - private val onlyEtagHashesMap = Option(Map( - "alfred" -> "xrd", - "betty" -> "abc123", - "charles" -> "456", - "etag" -> etagHashValue, - )) + private val completeHashesMap = Option( + Map( + "betty" -> "abc123", + "charles" -> "456", + "alfred" -> "xrd", + "sha256" -> shaHashValue, + "crc32c" -> crcHashValue, + "md5" -> md5HashValue, + "etag" -> etagHashValue + ) + ) + + private val missingCRCHashesMap = Option( + Map( + "alfred" -> "xrd", + "sha256" -> shaHashValue, + "betty" -> "abc123", + "md5" -> md5HashValue, + "charles" -> "456" + ) + ) + + private val onlySHAHashesMap = Option( + Map( + "betty" -> "abc123", + "charles" -> "456", + "alfred" -> "xrd", + "sha256" -> shaHashValue + ) + ) + + private val onlyEtagHashesMap = Option( + Map( + "alfred" -> "xrd", + "betty" -> "abc123", + "charles" -> "456", + "etag" -> etagHashValue + ) + ) behavior of "fileHash()" - it should "return crc32c hash from `hashes` in Martha response when there is a crc32c" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(completeHashesMap) shouldBe Option(FileHash(HashType.Crc32c, crcHashValue)) + it should "return crc32c hash from `hashes` in DRS Resolver response when there is a crc32c" in { + DrsCloudNioRegularFileAttributes.getPreferredHash(completeHashesMap) shouldBe Option( + FileHash(HashType.Crc32c, crcHashValue) + ) } - it should "return md5 hash from `hashes` in Martha response when there is no crc32c" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(missingCRCHashesMap) shouldBe Option(FileHash(HashType.Md5, md5HashValue)) + it should "return md5 hash from `hashes` in DRS Resolver response when there is no crc32c" in { + DrsCloudNioRegularFileAttributes.getPreferredHash(missingCRCHashesMap) shouldBe Option( + FileHash(HashType.Md5, md5HashValue) + ) } - it should "return sha256 hash from `hashes` in Martha response when there is only a sha256" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(onlySHAHashesMap) shouldBe Option(FileHash(HashType.Sha256, shaHashValue)) + it should "return sha256 hash from `hashes` in DRS Resolver response when there is only a sha256" in { + DrsCloudNioRegularFileAttributes.getPreferredHash(onlySHAHashesMap) shouldBe Option( + FileHash(HashType.Sha256, shaHashValue) + ) } - it should "return etag hash from `hashes` in Martha response when there is only an etag" in { - DrsCloudNioRegularFileAttributes.getPreferredHash(onlyEtagHashesMap) shouldBe Option(FileHash(HashType.S3Etag, etagHashValue)) + it should "return etag hash from `hashes` in DRS Resolver response when there is only an etag" in { + DrsCloudNioRegularFileAttributes.getPreferredHash(onlyEtagHashesMap) shouldBe Option( + FileHash(HashType.S3Etag, etagHashValue) + ) } it should "return None when no hashes object is returned" in { @@ -127,10 +144,10 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with it should "successfully parse a failure" in { import io.circe.parser.decode - import cloud.nio.impl.drs.MarthaResponseSupport.marthaFailureResponseDecoder + import cloud.nio.impl.drs.DrsResolverResponseSupport.drsResolverFailureResponseDecoder - val maybeDecoded = decode[MarthaFailureResponse](failureResponseJson) - maybeDecoded map { decoded: MarthaFailureResponse => + val maybeDecoded = decode[DrsResolverFailureResponse](failureResponseJson) + maybeDecoded map { decoded: DrsResolverFailureResponse => decoded.response.text shouldBe "{\"msg\":\"User 'null' does not have required action: read_data\",\"status_code\":500}" } } @@ -138,38 +155,62 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with import org.apache.http.message.BasicStatusLine val drsPathForDebugging = "drs://my_awesome_drs" - val responseStatusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 2) , 345, "test-reason") - val testMarthaUri = "www.martha_v3.com" + val responseStatusLine = new BasicStatusLine(new ProtocolVersion("http", 1, 2), 345, "test-reason") + val testDrsResolverUri = "www.drshub_v4.com" + + it should "construct the right request when using Azure creds" in { + val resolver = new MockDrsPathResolver(drsCredentials = AzureDrsCredentials()) + val drsRequest = resolver.makeDrsResolverRequest(drsPathForDebugging, NonEmptyList.of(DrsResolverField.AccessUrl)) + drsRequest.cloudPlatform shouldBe Option(DrsCloudPlatform.Azure) + } + + it should "construct the right request when using Google creds" in { + val resolver = new MockDrsPathResolver() + val drsRequest = resolver.makeDrsResolverRequest(drsPathForDebugging, NonEmptyList.of(DrsResolverField.AccessUrl)) + drsRequest.cloudPlatform shouldBe Option(DrsCloudPlatform.GoogleStorage) + } it should "construct an error message from a populated, well-formed failure response" in { val failureResponse = Option(failureResponseJson) - MarthaResponseSupport.errorMessageFromResponse(drsPathForDebugging, failureResponse, responseStatusLine, testMarthaUri) shouldBe { - "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', Martha location: 'www.martha_v3.com', message: '{\"msg\":\"User 'null' does not have required action: read_data\",\"status_code\":500}'" + DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, + failureResponse, + responseStatusLine, + testDrsResolverUri + ) shouldBe { + "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', DRS Resolver location: 'www.drshub_v4.com', message: '{\"msg\":\"User 'null' does not have required action: read_data\",\"status_code\":500}'" } } it should "construct an error message from an empty failure response" in { - MarthaResponseSupport.errorMessageFromResponse(drsPathForDebugging, None, responseStatusLine, testMarthaUri) shouldBe { - "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', Martha location: 'www.martha_v3.com', message: (empty response)" + DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, + None, + responseStatusLine, + testDrsResolverUri + ) shouldBe { + "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', DRS Resolver location: 'www.drshub_v4.com', message: (empty response)" } } - // Technically we enter this case when preparing the "error message" for a successful response, because naturally `MarthaResponse` does not deserialize to `MarthaFailureResponse` + // Technically we enter this case when preparing the "error message" for a successful response, because naturally `DrsResolverResponse` does not deserialize to `DrsResolverFailureResponse` // But then there's no error so we throw it away :shrug: it should "construct an error message from a malformed failure response" in { val unparsableFailureResponse = Option("something went horribly wrong") - MarthaResponseSupport.errorMessageFromResponse(drsPathForDebugging, unparsableFailureResponse, responseStatusLine, testMarthaUri) shouldBe { - "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', Martha location: 'www.martha_v3.com', message: 'something went horribly wrong'" + DrsResolverResponseSupport.errorMessageFromResponse(drsPathForDebugging, + unparsableFailureResponse, + responseStatusLine, + testDrsResolverUri + ) shouldBe { + "Could not access object 'drs://my_awesome_drs'. Status: 345, reason: 'test-reason', DRS Resolver location: 'www.drshub_v4.com', message: 'something went horribly wrong'" } } it should "resolve an ISO-8601 date with timezone" in { val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", - MarthaField.TimeUpdated, - fullMarthaResponse.timeUpdated, + DrsResolverField.TimeUpdated, + fullDrsResolverResponse.timeUpdated ) lastModifiedTimeIO.unsafeRunSync() should be(Option(FileTime.from(OffsetDateTime.parse("2020-04-27T15:56:09.696Z").toInstant))) @@ -178,8 +219,8 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with it should "resolve an ISO-8601 date without timezone" in { val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", - MarthaField.TimeUpdated, - fullMarthaResponseNoTz.timeUpdated, + DrsResolverField.TimeUpdated, + fullDrsResolverResponseNoTz.timeUpdated ) lastModifiedTimeIO.unsafeRunSync() should be(Option(FileTime.from(OffsetDateTime.parse("2020-04-27T15:56:09.696Z").toInstant))) @@ -188,8 +229,8 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with it should "not resolve an date that does not contain a timeUpdated" in { val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", - MarthaField.TimeUpdated, - fullMarthaResponseNoTime.timeUpdated, + DrsResolverField.TimeUpdated, + fullDrsResolverResponseNoTime.timeUpdated ) lastModifiedTimeIO.unsafeRunSync() should be(None) } @@ -197,11 +238,11 @@ class DrsPathResolverSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with it should "not resolve an date that is not ISO-8601" in { val lastModifiedTimeIO = convertToFileTime( "drs://my_awesome_drs", - MarthaField.TimeUpdated, - fullMarthaResponseBadTz.timeUpdated, + DrsResolverField.TimeUpdated, + fullDrsResolverResponseBadTz.timeUpdated ) the[RuntimeException] thrownBy lastModifiedTimeIO.unsafeRunSync() should have message - "Error while parsing 'timeUpdated' value from Martha to FileTime for DRS path drs://my_awesome_drs. " + - "Reason: DateTimeParseException: Text '2020-04-27T15:56:09.696BADTZ' could not be parsed at index 23." + "Error while parsing 'timeUpdated' value from DRS Resolver to FileTime for DRS path drs://my_awesome_drs. " + + "Reason: DateTimeParseException: Text '2020-04-27T15:56:09.696BADTZ' could not be parsed at index 23." } } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MarthaHttpRequestRetryStrategySpec.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategySpec.scala similarity index 88% rename from cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MarthaHttpRequestRetryStrategySpec.scala rename to cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategySpec.scala index 8fd1e948607..5be2b155391 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MarthaHttpRequestRetryStrategySpec.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/DrsResolverHttpRequestRetryStrategySpec.scala @@ -12,13 +12,13 @@ import common.mock.MockSugar import scala.concurrent.duration._ -class MarthaHttpRequestRetryStrategySpec extends AnyFlatSpec with Matchers with MockSugar { +class DrsResolverHttpRequestRetryStrategySpec extends AnyFlatSpec with Matchers with MockSugar { - behavior of "MarthaHttpRequestRetryStrategy" + behavior of "DrsResolverHttpRequestRetryStrategy" it should "retry 500 errors a configured number of times" in { val drsConfig = MockDrsPaths.mockDrsConfig.copy(numRetries = 3) - val retryStrategy = new MarthaHttpRequestRetryStrategy(drsConfig) + val retryStrategy = new DrsResolverHttpRequestRetryStrategy(drsConfig) val http500Response = mock[CloseableHttpResponse] http500Response.getStatusLine returns new BasicStatusLine(HttpVersion.HTTP_1_1, 500, "Testing 500") val httpContext = mock[HttpContext] @@ -35,7 +35,7 @@ class MarthaHttpRequestRetryStrategySpec extends AnyFlatSpec with Matchers with it should "retry 500 errors even after a number of 408/429 errors" in { val drsConfig = MockDrsPaths.mockDrsConfig.copy(numRetries = 3) - val retryStrategy = new MarthaHttpRequestRetryStrategy(drsConfig) + val retryStrategy = new DrsResolverHttpRequestRetryStrategy(drsConfig) val http500Response = mock[CloseableHttpResponse] http500Response.getStatusLine returns new BasicStatusLine(HttpVersion.HTTP_1_1, 500, "Testing 500") val http408Response = mock[CloseableHttpResponse] @@ -63,7 +63,7 @@ class MarthaHttpRequestRetryStrategySpec extends AnyFlatSpec with Matchers with it should "not retry an HTTP 401" in { val drsConfig = MockDrsPaths.mockDrsConfig.copy(numRetries = 3) - val retryStrategy = new MarthaHttpRequestRetryStrategy(drsConfig) + val retryStrategy = new DrsResolverHttpRequestRetryStrategy(drsConfig) val http400Response = mock[CloseableHttpResponse] http400Response.getStatusLine returns new BasicStatusLine(HttpVersion.HTTP_1_1, 401, "Testing 401") val httpContext = mock[HttpContext] @@ -73,7 +73,7 @@ class MarthaHttpRequestRetryStrategySpec extends AnyFlatSpec with Matchers with it should "retry IO exceptions a configured number of times" in { val drsConfig = MockDrsPaths.mockDrsConfig.copy(numRetries = 3) - val retryStrategy = new MarthaHttpRequestRetryStrategy(drsConfig) + val retryStrategy = new DrsResolverHttpRequestRetryStrategy(drsConfig) val exception = mock[IOException] val httpContext = mock[HttpContext] @@ -93,9 +93,9 @@ class MarthaHttpRequestRetryStrategySpec extends AnyFlatSpec with Matchers with waitInitial = 10.seconds, waitMultiplier = 2.0d, waitMaximum = 1.minute, - waitRandomizationFactor = 0d, + waitRandomizationFactor = 0d ) - val retryStrategy = new MarthaHttpRequestRetryStrategy(drsConfig) + val retryStrategy = new DrsResolverHttpRequestRetryStrategy(drsConfig) retryStrategy.getRetryInterval should be(10000L) retryStrategy.getRetryInterval should be(20000L) diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala index 025d7c6d543..5a2ba6172f1 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsCloudNioFileSystemProvider.scala @@ -7,31 +7,29 @@ import com.google.cloud.NoCredentials import com.typesafe.config.{Config, ConfigFactory} import org.apache.http.impl.client.HttpClientBuilder -import scala.concurrent.duration.Duration - class MockDrsCloudNioFileSystemProvider(config: Config = mockConfig, httpClientBuilder: Option[HttpClientBuilder] = None, drsReadInterpreter: DrsReadInterpreter = (_, _) => IO.raiseError( new UnsupportedOperationException("mock did not specify a read interpreter") ), - mockResolver: Option[EngineDrsPathResolver] = None, - ) - extends DrsCloudNioFileSystemProvider(config, GoogleDrsCredentials(NoCredentials.getInstance, config), drsReadInterpreter) { + mockResolver: Option[DrsPathResolver] = None +) extends DrsCloudNioFileSystemProvider(config, + GoogleOauthDrsCredentials(NoCredentials.getInstance, config), + drsReadInterpreter + ) { - override lazy val drsPathResolver: EngineDrsPathResolver = { + override lazy val drsPathResolver: DrsPathResolver = mockResolver getOrElse - new MockEngineDrsPathResolver( + new MockDrsPathResolver( drsConfig = drsConfig, - httpClientBuilderOverride = httpClientBuilder, - accessTokenAcceptableTTL = Duration.Inf, + httpClientBuilderOverride = httpClientBuilder ) - } } object MockDrsCloudNioFileSystemProvider { private lazy val mockConfig = ConfigFactory.parseString( - """martha.url = "https://mock.martha" + """resolver.url = "https://mock.drshub" |access-token-acceptable-ttl = 1 hour |""".stripMargin ) diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPathResolver.scala new file mode 100644 index 00000000000..9439c7d94ab --- /dev/null +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPathResolver.scala @@ -0,0 +1,67 @@ +package cloud.nio.impl.drs + +import cats.data.NonEmptyList +import cats.effect.IO +import cats.syntax.validated._ +import com.google.cloud.NoCredentials +import common.validation.ErrorOr.ErrorOr +import org.apache.http.impl.client.HttpClientBuilder +import common.mock.MockSugar + +import scala.concurrent.duration.Duration + +class MockDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfig, + httpClientBuilderOverride: Option[HttpClientBuilder] = None, + drsCredentials: DrsCredentials = + GoogleOauthDrsCredentials(NoCredentials.getInstance, Duration.Inf) +) extends DrsPathResolver(drsConfig, drsCredentials) { + + override protected lazy val httpClientBuilder: HttpClientBuilder = + httpClientBuilderOverride getOrElse MockSugar.mock[HttpClientBuilder] + + private lazy val mockDrsResolverUri = drsConfig.drsResolverUrl + + private val hashesObj = Map( + "md5" -> "336ea55913bc261b72875bd259753046", + "sha256" -> "f76877f8e86ec3932fd2ae04239fbabb8c90199dab0019ae55fa42b31c314c44", + "crc32c" -> "8a366443" + ) + + private val drsResolverObjWithGcsPath = + DrsResolverResponse( + size = Option(156018255), + timeCreated = Option("2020-04-27T15:56:09.696Z"), + timeUpdated = Option("2020-04-27T15:56:09.696Z"), + gsUri = Option(s"gs://${MockDrsPaths.drsRelativePath}"), + hashes = Option(hashesObj) + ) + + private val drsResolverObjWithFileName = drsResolverObjWithGcsPath.copy(fileName = Option("file.txt")) + + private val drsResolverObjWithLocalizationPath = + drsResolverObjWithGcsPath.copy(localizationPath = Option("/dir/subdir/file.txt")) + + private val drsResolverObjWithAllThePaths = + drsResolverObjWithLocalizationPath.copy(fileName = drsResolverObjWithFileName.fileName) + + private val drsResolverObjWithNoGcsPath = drsResolverObjWithGcsPath.copy(gsUri = None) + + override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = + drsPath match { + case MockDrsPaths.drsPathResolvingGcsPath => IO(drsResolverObjWithGcsPath) + case MockDrsPaths.drsPathWithNonPathChars => IO(drsResolverObjWithGcsPath) + case MockDrsPaths.drsPathResolvingWithFileName => IO(drsResolverObjWithFileName) + case MockDrsPaths.drsPathResolvingWithLocalizationPath => IO.pure(drsResolverObjWithLocalizationPath) + case MockDrsPaths.drsPathResolvingWithAllThePaths => IO.pure(drsResolverObjWithAllThePaths) + case MockDrsPaths.drsPathResolvingToNoGcsPath => IO(drsResolverObjWithNoGcsPath) + case MockDrsPaths.`drsPathNotExistingInDrsResolver` => + IO.raiseError( + new RuntimeException( + s"Unexpected response resolving ${MockDrsPaths.drsPathNotExistingInDrsResolver} " + + s"through DRS Resolver url $mockDrsResolverUri. Error: 404 Not Found." + ) + ) + } + + override lazy val getAccessToken: ErrorOr[String] = MockDrsPaths.mockToken.validNel +} diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala index b6825dcbbd2..a24385bc15f 100644 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala +++ b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockDrsPaths.scala @@ -3,9 +3,9 @@ package cloud.nio.impl.drs import scala.concurrent.duration._ object MockDrsPaths { - val marthaUrl = "http://mock.martha" + val drsResolverUrl = "http://mock.drshub" - val mockDrsConfig: DrsConfig = DrsConfig(MockDrsPaths.marthaUrl, 1, 1.seconds, 1.seconds, 1d, 0d) + val mockDrsConfig: DrsConfig = DrsConfig(MockDrsPaths.drsResolverUrl, 1, 1.seconds, 1.seconds, 1d, 0d) val mockToken = "mock.token" @@ -35,5 +35,5 @@ object MockDrsPaths { val drsPathResolvingToNoGcsPath = s"$drsPathPrefix/226686cf-22c9-4472-9f79-7a0b0044f253" - val drsPathNotExistingInMartha = s"$drsPathPrefix/5e21b8c3-8eda-48d5-9a04-2b32e1571765" + val drsPathNotExistingInDrsResolver = s"$drsPathPrefix/5e21b8c3-8eda-48d5-9a04-2b32e1571765" } diff --git a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala b/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala deleted file mode 100644 index 116155753f1..00000000000 --- a/cloud-nio/cloud-nio-impl-drs/src/test/scala/cloud/nio/impl/drs/MockEngineDrsPathResolver.scala +++ /dev/null @@ -1,66 +0,0 @@ -package cloud.nio.impl.drs - -import cats.data.NonEmptyList -import cats.effect.IO -import cats.syntax.validated._ -import com.google.cloud.NoCredentials -import common.validation.ErrorOr.ErrorOr -import org.apache.http.impl.client.HttpClientBuilder -import common.mock.MockSugar - -import scala.concurrent.duration.Duration - -class MockEngineDrsPathResolver(drsConfig: DrsConfig = MockDrsPaths.mockDrsConfig, - httpClientBuilderOverride: Option[HttpClientBuilder] = None, - accessTokenAcceptableTTL: Duration = Duration.Inf, - ) - extends EngineDrsPathResolver(drsConfig, GoogleDrsCredentials(NoCredentials.getInstance, accessTokenAcceptableTTL)) { - - override protected lazy val httpClientBuilder: HttpClientBuilder = - httpClientBuilderOverride getOrElse MockSugar.mock[HttpClientBuilder] - - private lazy val mockMarthaUri = drsConfig.marthaUrl - - private val hashesObj = Map( - "md5" -> "336ea55913bc261b72875bd259753046", - "sha256" -> "f76877f8e86ec3932fd2ae04239fbabb8c90199dab0019ae55fa42b31c314c44", - "crc32c" -> "8a366443" - ) - - private val marthaObjWithGcsPath = - MarthaResponse( - size = Option(156018255), - timeCreated = Option("2020-04-27T15:56:09.696Z"), - timeUpdated = Option("2020-04-27T15:56:09.696Z"), - gsUri = Option(s"gs://${MockDrsPaths.drsRelativePath}"), - hashes = Option(hashesObj) - ) - - private val marthaObjWithFileName = marthaObjWithGcsPath.copy(fileName = Option("file.txt")) - - private val marthaObjWithLocalizationPath = marthaObjWithGcsPath.copy(localizationPath = Option("/dir/subdir/file.txt")) - - private val marthaObjWithAllThePaths = marthaObjWithLocalizationPath.copy(fileName = marthaObjWithFileName.fileName) - - private val marthaObjWithNoGcsPath = marthaObjWithGcsPath.copy(gsUri = None) - - override def resolveDrsThroughMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): IO[MarthaResponse] = { - drsPath match { - case MockDrsPaths.drsPathResolvingGcsPath => IO(marthaObjWithGcsPath) - case MockDrsPaths.drsPathWithNonPathChars => IO(marthaObjWithGcsPath) - case MockDrsPaths.drsPathResolvingWithFileName => IO(marthaObjWithFileName) - case MockDrsPaths.drsPathResolvingWithLocalizationPath => IO.pure(marthaObjWithLocalizationPath) - case MockDrsPaths.drsPathResolvingWithAllThePaths => IO.pure(marthaObjWithAllThePaths) - case MockDrsPaths.drsPathResolvingToNoGcsPath => IO(marthaObjWithNoGcsPath) - case MockDrsPaths.drsPathNotExistingInMartha => - IO.raiseError( - new RuntimeException( - s"Unexpected response resolving ${MockDrsPaths.drsPathNotExistingInMartha} " + - s"through Martha url $mockMarthaUri. Error: 404 Not Found." - ) - ) - } - } - - override lazy val getAccessToken: ErrorOr[String] = MockDrsPaths.mockToken.validNel -} diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala index d93b0aa6f82..b13eb844a24 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpClientPool.scala @@ -7,25 +7,26 @@ import org.apache.commons.net.ftp.FTPClient import scala.concurrent.duration._ object FtpClientPool extends StrictLogging { - def dispose(ftpClient: FTPClient) = try { + def dispose(ftpClient: FTPClient) = try if (ftpClient.isConnected) { ftpClient.logout() ftpClient.disconnect() } - } catch { + catch { case e: Exception => logger.debug("Failed to disconnect ftp client", e) } } -class FtpClientPool(capacity: Int, maxIdleTime: FiniteDuration, factory: () => FTPClient) extends ExpiringPool[FTPClient]( - capacity = capacity, - maxIdleTime = maxIdleTime, - referenceType = ReferenceType.Strong, - _factory = factory, - // Reset is called every time a client is added or released back to the pool. We don't want to actually reset the connection here - // otherwise we'd need to login again and reconfigure the connection every time - _reset = Function.const(()), - _dispose = FtpClientPool.dispose, - // Could not find a good health check at the moment (isAvailable and isConnected on the socket seem to both return false sometimes even if the client is fine) - _healthCheck = Function.const(true) -) +class FtpClientPool(capacity: Int, maxIdleTime: FiniteDuration, factory: () => FTPClient) + extends ExpiringPool[FTPClient]( + capacity = capacity, + maxIdleTime = maxIdleTime, + referenceType = ReferenceType.Strong, + _factory = factory, + // Reset is called every time a client is added or released back to the pool. We don't want to actually reset the connection here + // otherwise we'd need to login again and reconfigure the connection every time + _reset = Function.const(()), + _dispose = FtpClientPool.dispose, + // Could not find a good health check at the moment (isAvailable and isConnected on the socket seem to both return false sometimes even if the client is fine) + _healthCheck = Function.const(true) + ) diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala index 96e05702034..37ace6c250b 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileProvider.scala @@ -37,17 +37,25 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends * Returns a listing of keys within a bucket starting with prefix. The returned keys should include the prefix. The * paths must be absolute, but the key should not begin with a slash. */ - override def listObjects(cloudHost: String, cloudPathPrefix: String, markerOption: Option[String]): CloudNioFileList = withAutoRelease(cloudHost) { client => - FtpListFiles(cloudHost, cloudPathPrefix, "list objects") - .run(client) - .map({ files => - val cleanFiles = files.map(_.getName).map(cloudPathPrefix.stripPrefix("/").ensureSlashed + _) - CloudNioFileList(cleanFiles, markerOption) - }) - }.unsafeRunSync() - - override def copy(sourceCloudHost: String, sourceCloudPath: String, targetCloudHost: String, targetCloudPath: String): Unit = { - if (sourceCloudHost != targetCloudHost) throw new UnsupportedOperationException(s"Cannot copy files across different ftp servers: Source host: $sourceCloudHost, Target host: $targetCloudHost") + override def listObjects(cloudHost: String, cloudPathPrefix: String, markerOption: Option[String]): CloudNioFileList = + withAutoRelease(cloudHost) { client => + FtpListFiles(cloudHost, cloudPathPrefix, "list objects") + .run(client) + .map { files => + val cleanFiles = files.map(_.getName).map(cloudPathPrefix.stripPrefix("/").ensureSlashed + _) + CloudNioFileList(cleanFiles, markerOption) + } + }.unsafeRunSync() + + override def copy(sourceCloudHost: String, + sourceCloudPath: String, + targetCloudHost: String, + targetCloudPath: String + ): Unit = { + if (sourceCloudHost != targetCloudHost) + throw new UnsupportedOperationException( + s"Cannot copy files across different ftp servers: Source host: $sourceCloudHost, Target host: $targetCloudHost" + ) val fileSystem = findFileSystem(sourceCloudHost) @@ -63,7 +71,11 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends Util.copyStream(ios.inputStream, ios.outputStream) } match { case Success(_) => - case Failure(failure) => throw new IOException(s"Failed to copy ftp://$sourceCloudHost/$sourceCloudPath to ftp://$targetCloudHost/$targetCloudPath", failure) + case Failure(failure) => + throw new IOException( + s"Failed to copy ftp://$sourceCloudHost/$sourceCloudPath to ftp://$targetCloudHost/$targetCloudPath", + failure + ) } } @@ -71,12 +83,15 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends FtpDeleteFile(cloudHost, cloudPath, "delete").run(client) }.unsafeRunSync() - private def inputStream(cloudHost: String, cloudPath: String, offset: Long, lease: Lease[FTPClient]): IO[LeasedInputStream] = { + private def inputStream(cloudHost: String, + cloudPath: String, + offset: Long, + lease: Lease[FTPClient] + ): IO[LeasedInputStream] = FtpInputStream(cloudHost, cloudPath, offset) .run(lease.get()) // Wrap the input stream in a LeasedInputStream so that the lease can be released when the stream is closed .map(new LeasedInputStream(cloudHost, cloudPath, _, lease)) - } override def read(cloudHost: String, cloudPath: String, offset: Long): ReadableByteChannel = { for { @@ -85,11 +100,10 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends } yield Channels.newChannel(is) }.unsafeRunSync() - private def outputStream(cloudHost: String, cloudPath: String, lease: Lease[FTPClient]): IO[LeasedOutputStream] = { + private def outputStream(cloudHost: String, cloudPath: String, lease: Lease[FTPClient]): IO[LeasedOutputStream] = FtpOutputStream(cloudHost, cloudPath) .run(lease.get()) .map(new LeasedOutputStream(cloudHost, cloudPath, _, lease)) - } override def write(cloudHost: String, cloudPath: String): WritableByteChannel = { for { @@ -98,15 +112,16 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends } yield Channels.newChannel(os) }.unsafeRunSync() - override def fileAttributes(cloudHost: String, cloudPath: String): Option[CloudNioRegularFileAttributes] = withAutoRelease(cloudHost) { client => - FtpListFiles(cloudHost, cloudPath, "get file attributes") - .run(client) - .map( - _.headOption map { file => - new FtpCloudNioRegularFileAttributes(file, cloudHost + cloudPath) - } - ) - }.unsafeRunSync() + override def fileAttributes(cloudHost: String, cloudPath: String): Option[CloudNioRegularFileAttributes] = + withAutoRelease(cloudHost) { client => + FtpListFiles(cloudHost, cloudPath, "get file attributes") + .run(client) + .map( + _.headOption map { file => + new FtpCloudNioRegularFileAttributes(file, cloudHost + cloudPath) + } + ) + }.unsafeRunSync() override def createDirectory(cloudHost: String, cloudPath: String) = withAutoRelease(cloudHost) { client => val operation = FtpCreateDirectory(cloudHost, cloudPath) @@ -129,7 +144,8 @@ class FtpCloudNioFileProvider(fsProvider: FtpCloudNioFileSystemProvider) extends private def findFileSystem(host: String): FtpCloudNioFileSystem = fsProvider.newCloudNioFileSystemFromHost(host) - private def acquireLease[A](host: String): IO[Lease[FTPClient]] = IO { findFileSystem(host).leaseClient } + private def acquireLease[A](host: String): IO[Lease[FTPClient]] = IO(findFileSystem(host).leaseClient) - private def withAutoRelease[A](cloudHost: String): (FTPClient => IO[A]) => IO[A] = autoRelease[A](acquireLease(cloudHost)) + private def withAutoRelease[A](cloudHost: String): (FTPClient => IO[A]) => IO[A] = + autoRelease[A](acquireLease(cloudHost)) } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala index 91a26bc8f8e..dd0bb79cfa7 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystem.scala @@ -12,7 +12,9 @@ object FtpCloudNioFileSystem { val logger = LoggerFactory.getLogger("FtpFileSystem") } -class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: String) extends CloudNioFileSystem(provider, host) with StrictLogging { +class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: String) + extends CloudNioFileSystem(provider, host) + with StrictLogging { private val credentials = provider.credentials private val ftpConfig = provider.ftpConfig private val connectionModeFunction: FTPClient => Unit = ftpConfig.connectionMode match { @@ -20,7 +22,7 @@ class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: Strin case Active => client: FTPClient => client.enterLocalActiveMode() } - private [ftp] lazy val clientFactory = () => { + private[ftp] lazy val clientFactory = () => { val client = new FTPClient() client.setDefaultPort(ftpConfig.connectionPort) client.connect(host) @@ -33,7 +35,10 @@ class FtpCloudNioFileSystem(provider: FtpCloudNioFileSystemProvider, host: Strin private val clientPool = new FtpClientPool(ftpConfig.capacity, ftpConfig.idleConnectionTimeout, clientFactory) def leaseClient = ftpConfig.leaseTimeout match { - case Some(timeout) => clientPool.tryAcquire(timeout).getOrElse(throw new TimeoutException("Timed out waiting for an available connection, try again later.")) + case Some(timeout) => + clientPool + .tryAcquire(timeout) + .getOrElse(throw new TimeoutException("Timed out waiting for an available connection, try again later.")) case _ => clientPool.acquire() } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala index e20edfc76ff..bb2faaa5ccd 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProvider.scala @@ -11,7 +11,11 @@ import com.typesafe.scalalogging.StrictLogging import scala.util.{Failure, Success, Try} -class FtpCloudNioFileSystemProvider(override val config: Config, val credentials: FtpCredentials, ftpFileSystems: FtpFileSystems) extends CloudNioFileSystemProvider with StrictLogging { +class FtpCloudNioFileSystemProvider(override val config: Config, + val credentials: FtpCredentials, + ftpFileSystems: FtpFileSystems +) extends CloudNioFileSystemProvider + with StrictLogging { val ftpConfig = ftpFileSystems.config override def fileProvider = new FtpCloudNioFileProvider(this) @@ -29,18 +33,19 @@ class FtpCloudNioFileSystemProvider(override val config: Config, val credentials * will try to get it using the fileProvider which will require a new client lease and can result in a deadlock of the client pool, since * the read channel holds on to its lease until its closed. */ - val preComputedFileSize = retry.from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath).map(_.size())) + val preComputedFileSize = + retry.from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath).map(_.size())) new CloudNioReadChannel(fileProvider, retry, cloudNioPath) { override def fileSize = preComputedFileSize } } - override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = { + override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = Try { - retry.from(() => { + retry.from { () => val cloudNioPath = CloudNioPath.checkPath(dir) fileProvider.createDirectory(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - }) + } } match { case Success(_) => case Failure(f: FileAlreadyExistsException) => throw f @@ -51,15 +56,12 @@ class FtpCloudNioFileSystemProvider(override val config: Config, val credentials throw f case Failure(f) => throw f } - } override def usePseudoDirectories = false - override def newCloudNioFileSystem(uriAsString: String, config: Config) = { + override def newCloudNioFileSystem(uriAsString: String, config: Config) = newCloudNioFileSystemFromHost(getHost(uriAsString)) - } - - def newCloudNioFileSystemFromHost(host: String) = { + + def newCloudNioFileSystemFromHost(host: String) = ftpFileSystems.getFileSystem(host, this) - } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala index 2e553a5f66b..c63efad1de7 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpCredentials.scala @@ -11,7 +11,8 @@ sealed trait FtpCredentials { } // Yes, FTP uses plain text username / password -case class FtpAuthenticatedCredentials(username: String, password: String, account: Option[String]) extends FtpCredentials { +case class FtpAuthenticatedCredentials(username: String, password: String, account: Option[String]) + extends FtpCredentials { override def login(ftpClient: FTPClient) = { lazy val replyString = Option(ftpClient.getReplyString).getOrElse("N/A") diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala index 006e19982ff..63f0c10f655 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystems.scala @@ -9,7 +9,7 @@ import scala.concurrent.duration._ object FtpFileSystems { val DefaultConfig = FtpFileSystemsConfiguration(1.day, Option(1.hour), 5, 1.hour, 21, Passive) val Default = new FtpFileSystems(DefaultConfig) - private [ftp] case class FtpCacheKey(host: String, ftpProvider: FtpCloudNioFileSystemProvider) + private[ftp] case class FtpCacheKey(host: String, ftpProvider: FtpCloudNioFileSystemProvider) } /** @@ -21,18 +21,18 @@ class FtpFileSystems(val config: FtpFileSystemsConfiguration) { private val fileSystemTTL = config.cacheTTL - private val fileSystemsCache: LoadingCache[FtpCacheKey, FtpCloudNioFileSystem] = CacheBuilder.newBuilder() + private val fileSystemsCache: LoadingCache[FtpCacheKey, FtpCloudNioFileSystem] = CacheBuilder + .newBuilder() .expireAfterAccess(fileSystemTTL.length, fileSystemTTL.unit) - .removalListener((notification: RemovalNotification[FtpCacheKey, FtpCloudNioFileSystem]) => { + .removalListener { (notification: RemovalNotification[FtpCacheKey, FtpCloudNioFileSystem]) => notification.getValue.close() - }) + } .build[FtpCacheKey, FtpCloudNioFileSystem](new CacheLoader[FtpCacheKey, FtpCloudNioFileSystem] { override def load(key: FtpCacheKey) = createFileSystem(key) }) - - private [ftp] def createFileSystem(key: FtpCacheKey) = new FtpCloudNioFileSystem(key.ftpProvider, key.host) - def getFileSystem(host: String, ftpCloudNioFileProvider: FtpCloudNioFileSystemProvider) = { + private[ftp] def createFileSystem(key: FtpCacheKey) = new FtpCloudNioFileSystem(key.ftpProvider, key.host) + + def getFileSystem(host: String, ftpCloudNioFileProvider: FtpCloudNioFileSystemProvider) = fileSystemsCache.get(FtpCacheKey(host, ftpCloudNioFileProvider)) - } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala index 2c305f82edf..8e5071f604a 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpFileSystemsConfiguration.scala @@ -9,7 +9,8 @@ case class FtpFileSystemsConfiguration(cacheTTL: FiniteDuration, capacity: Int, idleConnectionTimeout: FiniteDuration, connectionPort: Int, - connectionMode: ConnectionMode) + connectionMode: ConnectionMode +) object FtpFileSystemsConfiguration { sealed trait ConnectionMode diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala index 9138401f803..049c54d9d89 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/FtpUtil.scala @@ -12,16 +12,16 @@ object FtpUtil { def ensureSlashed = if (cloudPath.endsWith("/")) cloudPath else s"$cloudPath/" } - case class FtpIoException(message: String, code: Int, replyString: String, cause: Option[Throwable] = None) extends IOException(s"$message: $replyString", cause.orNull) { + case class FtpIoException(message: String, code: Int, replyString: String, cause: Option[Throwable] = None) + extends IOException(s"$message: $replyString", cause.orNull) { def isTransient = FTPReply.isNegativeTransient(code) def isFatal = FTPReply.isNegativePermanent(code) } - def autoRelease[A](acquire: IO[Lease[FTPClient]])(action: FTPClient => IO[A]): IO[A] = { - acquire.bracketCase(lease => action(lease.get()))({ - // If there's a cause, the call to the FTP client threw an exception, assume the connection is compromised and invalidate the lease - case (lease, ExitCase.Error(FtpIoException(_, _, _, Some(_)))) => IO { lease.invalidate() } - case (lease, _) => IO { lease.release() } - }) - } + def autoRelease[A](acquire: IO[Lease[FTPClient]])(action: FTPClient => IO[A]): IO[A] = + acquire.bracketCase(lease => action(lease.get())) { + // If there's a cause, the call to the FTP client threw an exception, assume the connection is compromised and invalidate the lease + case (lease, ExitCase.Error(FtpIoException(_, _, _, Some(_)))) => IO(lease.invalidate()) + case (lease, _) => IO(lease.release()) + } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala index 2455fa2b86a..a1a8b158ec7 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/InputOutputStreams.scala @@ -3,11 +3,9 @@ package cloud.nio.impl.ftp import java.io.{InputStream, OutputStream} class InputOutputStreams(val inputStream: InputStream, val outputStream: OutputStream) extends AutoCloseable { - override def close() = { - try { + override def close() = + try inputStream.close() - } finally { + finally outputStream.close() - } - } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala index 29d0b7c6bf7..2e794a96791 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedInputStream.scala @@ -8,7 +8,8 @@ import cloud.nio.impl.ftp.operations.FtpCompletePendingCommand import io.github.andrebeat.pool.Lease import org.apache.commons.net.ftp.FTPClient -class LeasedInputStream(cloudHost: String, cloudPath: String, inputStream: InputStream, lease: Lease[FTPClient]) extends InputStream { +class LeasedInputStream(cloudHost: String, cloudPath: String, inputStream: InputStream, lease: Lease[FTPClient]) + extends InputStream { override def read() = inputStream.read() override def read(b: Array[Byte]) = inputStream.read(b) override def read(b: Array[Byte], off: Int, len: Int): Int = inputStream.read(b, off, len) @@ -16,7 +17,8 @@ class LeasedInputStream(cloudHost: String, cloudPath: String, inputStream: Input override def available = inputStream.available() override def close() = { inputStream.close() - autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void.unsafeRunSync() + autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void + .unsafeRunSync() } override def mark(readlimit: Int) = inputStream.mark(readlimit) override def reset() = inputStream.reset() diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala index 5a4f43acabb..7fe09f3fbf1 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/LeasedOutputStream.scala @@ -8,13 +8,15 @@ import cloud.nio.impl.ftp.operations.FtpCompletePendingCommand import io.github.andrebeat.pool.Lease import org.apache.commons.net.ftp.FTPClient -class LeasedOutputStream(cloudHost: String, cloudPath: String, outputStream: OutputStream, lease: Lease[FTPClient]) extends OutputStream { +class LeasedOutputStream(cloudHost: String, cloudPath: String, outputStream: OutputStream, lease: Lease[FTPClient]) + extends OutputStream { override def write(b: Int) = outputStream.write(b) override def write(b: Array[Byte]) = outputStream.write(b) override def write(b: Array[Byte], off: Int, len: Int): Unit = outputStream.write(b, off, len) override def flush() = outputStream.flush() override def close() = { outputStream.close() - autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void.unsafeRunSync() + autoRelease(IO.pure(lease))(FtpCompletePendingCommand(cloudHost, cloudPath, "close input steam").run).void + .unsafeRunSync() } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala index 400b0b26740..dee465cff4c 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/main/scala/cloud/nio/impl/ftp/operations/FtpOperation.scala @@ -14,93 +14,101 @@ sealed trait FtpOperation[A] { def description: String def action: FTPClient => A def run(client: FTPClient): IO[A] - + def fullPath = s"ftp://$cloudHost/$cloudPath" protected def errorMessage = s"Failed to $description at $fullPath" - protected def fail(client: FTPClient, cause: Option[Throwable] = None) = { + protected def fail(client: FTPClient, cause: Option[Throwable] = None) = IO.raiseError[A](generateException(client, cause)) - } - private [operations] def generateException(client: FTPClient, cause: Option[Throwable]) = cause match { - case None if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains("exists") => + private[operations] def generateException(client: FTPClient, cause: Option[Throwable]) = cause match { + case None + if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains("exists") => new FileAlreadyExistsException(fullPath) - case None if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains("no such file") => + case None + if client.getReplyCode == FTPReply.FILE_UNAVAILABLE && client.getReplyString.toLowerCase.contains( + "no such file" + ) => new NoSuchFileException(fullPath) case None => FtpIoException(errorMessage, client.getReplyCode, Option(client.getReplyString).getOrElse("N/A")) case Some(c) => FtpIoException(errorMessage, client.getReplyCode, Option(client.getReplyString).getOrElse("N/A"), Option(c)) } - + protected def handleError(client: FTPClient)(failure: Throwable) = fail(client, Option(failure)) - protected def commonRun(client: FTPClient, bind: A => IO[A]): IO[A] = { - IO { action(client) } redeemWith (handleError(client), bind) - } - + protected def commonRun(client: FTPClient, bind: A => IO[A]): IO[A] = + IO(action(client)) redeemWith (handleError(client), bind) + override def toString = s"$description at $fullPath" } sealed trait FtpBooleanOperation extends FtpOperation[Boolean] { protected def failOnFalse: Boolean = true - - def run(client: FTPClient): IO[Boolean] = { + + def run(client: FTPClient): IO[Boolean] = commonRun(client, - { - // Operation didn't throw but the result is false which means it failed - case false if failOnFalse => fail(client) - case result => IO.pure(result) - } + { + // Operation didn't throw but the result is false which means it failed + case false if failOnFalse => fail(client) + case result => IO.pure(result) + } ) - } } sealed trait FtpValueOperation[A <: AnyRef] extends FtpOperation[A] { - def run(client: FTPClient): IO[A] = { - commonRun(client, - { - // Operation didn't throw but the result is null which means it failed - case null => fail(client) - case result => IO.pure(result) - } + def run(client: FTPClient): IO[A] = + commonRun(client, + { + // Operation didn't throw but the result is null which means it failed + case null => fail(client) + case result => IO.pure(result) + } ) - } } -case class FtpListFiles(cloudHost: String, cloudPath: String, description: String = "List files") extends FtpValueOperation[Array[FTPFile]] { +case class FtpListFiles(cloudHost: String, cloudPath: String, description: String = "List files") + extends FtpValueOperation[Array[FTPFile]] { override val action = _.listFiles(cloudPath.ensureSlashedPrefix) } -case class FtpListDirectories(cloudHost: String, cloudPath: String, description: String = "List files") extends FtpValueOperation[Array[FTPFile]] { +case class FtpListDirectories(cloudHost: String, cloudPath: String, description: String = "List files") + extends FtpValueOperation[Array[FTPFile]] { // We need to list the directories in the parent and see if any matches the name, hence the string manipulations - lazy val parts = cloudPath.ensureSlashedPrefix.stripSuffix(CloudNioFileSystem.Separator).split(CloudNioFileSystem.Separator) + lazy val parts = + cloudPath.ensureSlashedPrefix.stripSuffix(CloudNioFileSystem.Separator).split(CloudNioFileSystem.Separator) lazy val parent = parts.init.mkString(CloudNioFileSystem.Separator) lazy val directoryName = parts.last override val action = _.listDirectories(parent) } -case class FtpDeleteFile(cloudHost: String, cloudPath: String, description: String = "delete file") extends FtpBooleanOperation { +case class FtpDeleteFile(cloudHost: String, cloudPath: String, description: String = "delete file") + extends FtpBooleanOperation { override val action = _.deleteFile(cloudPath.ensureSlashedPrefix) override val failOnFalse = false } -case class FtpInputStream(cloudHost: String, cloudPath: String, offset: Long, description: String = "read") extends FtpValueOperation[InputStream] { +case class FtpInputStream(cloudHost: String, cloudPath: String, offset: Long, description: String = "read") + extends FtpValueOperation[InputStream] { override val action = { client => client.setRestartOffset(offset) client.retrieveFileStream(cloudPath.ensureSlashedPrefix) } } -case class FtpOutputStream(cloudHost: String, cloudPath: String, description: String = "write") extends FtpValueOperation[OutputStream] { +case class FtpOutputStream(cloudHost: String, cloudPath: String, description: String = "write") + extends FtpValueOperation[OutputStream] { override val action = _.storeFileStream(cloudPath.ensureSlashedPrefix) } -case class FtpCreateDirectory(cloudHost: String, cloudPath: String, description: String = "create directory") extends FtpBooleanOperation { +case class FtpCreateDirectory(cloudHost: String, cloudPath: String, description: String = "create directory") + extends FtpBooleanOperation { override val action = _.makeDirectory(cloudPath.ensureSlashedPrefix) } -case class FtpCompletePendingCommand(cloudHost: String, cloudPath: String, description: String = "close stream") extends FtpBooleanOperation { +case class FtpCompletePendingCommand(cloudHost: String, cloudPath: String, description: String = "close stream") + extends FtpBooleanOperation { override val action = _.completePendingCommand() } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala index 91b69bf7e94..1837ee27139 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpClientPoolSpec.scala @@ -17,15 +17,13 @@ class FtpClientPoolSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche var loggedOut: Boolean = false var disconnected: Boolean = false client.isConnected.returns(true) - client.logout().responds(_ => { + client.logout().responds { _ => loggedOut = true true - }) - client.disconnect().responds(_ => { - disconnected = true - }) + } + client.disconnect().responds(_ => disconnected = true) - val clientPool = new FtpClientPool(1, 10.minutes, () => { client }) + val clientPool = new FtpClientPool(1, 10.minutes, () => client) clientPool.acquire().invalidate() loggedOut shouldBe true diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala index f68f67cbf67..f1ca24c847a 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileProviderSpec.scala @@ -46,7 +46,8 @@ class FtpCloudNioFileProviderSpec extends AnyFlatSpec with CromwellTimeoutSpec w fakeUnixFileSystem.add(new DirectoryEntry(directory)) fileProvider.listObjects("localhost", root, None).paths should contain theSameElementsAs List( - file.stripPrefix("/"), directory.stripPrefix("/") + file.stripPrefix("/"), + directory.stripPrefix("/") ) } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala index e23e8ba450e..952e171f9ec 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemProviderSpec.scala @@ -13,8 +13,12 @@ import org.mockito.Mockito._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -class FtpCloudNioFileSystemProviderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with MockSugar - with MockFtpFileSystem { +class FtpCloudNioFileSystemProviderSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with MockSugar + with MockFtpFileSystem { behavior of "FtpCloudNioFileSystemProviderSpec" @@ -45,7 +49,9 @@ class FtpCloudNioFileSystemProviderSpec extends AnyFlatSpec with CromwellTimeout it should "pre compute the size before opening a read channel to avoid deadlocks" in { val mockSizeFunction = mock[() => Long] val provider: FtpCloudNioFileSystemProvider = new FtpCloudNioFileSystemProvider( - ConfigFactory.empty, FtpAnonymousCredentials, ftpFileSystems + ConfigFactory.empty, + FtpAnonymousCredentials, + ftpFileSystems ) { override def fileProvider: FtpCloudNioFileProvider = new FtpCloudNioFileProvider(this) { @@ -59,9 +65,8 @@ class FtpCloudNioFileSystemProviderSpec extends AnyFlatSpec with CromwellTimeout } ) - override def read(cloudHost: String, cloudPath: String, offset: Long): ReadableByteChannel = { + override def read(cloudHost: String, cloudPath: String, offset: Long): ReadableByteChannel = mock[ReadableByteChannel] - } } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala index d12db7ec612..f2b86fbd621 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCloudNioFileSystemSpec.scala @@ -11,7 +11,6 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.TimeoutException import scala.concurrent.duration._ - class FtpCloudNioFileSystemSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with Eventually { behavior of "FtpCloudNioFileSystemSpec" @@ -20,11 +19,12 @@ class FtpCloudNioFileSystemSpec extends AnyFlatSpec with CromwellTimeoutSpec wit implicit val patience = patienceConfig it should "lease the number of clients configured, not more, not less" in { - val fileSystems = new FtpFileSystems(FtpFileSystems.DefaultConfig.copy(leaseTimeout = Option(1.second), capacity = 3)) + val fileSystems = + new FtpFileSystems(FtpFileSystems.DefaultConfig.copy(leaseTimeout = Option(1.second), capacity = 3)) val provider = new FtpCloudNioFileSystemProvider(ConfigFactory.empty, FtpAnonymousCredentials, fileSystems) val fileSystem = new FtpCloudNioFileSystem(provider, "ftp.example.com") { // Override so we don't try to connect to anything - override private[ftp] lazy val clientFactory = () => { new FTPClient() } + override private[ftp] lazy val clientFactory = () => new FTPClient() } val client1 = fileSystem.leaseClient @@ -46,7 +46,7 @@ class FtpCloudNioFileSystemSpec extends AnyFlatSpec with CromwellTimeoutSpec wit val provider = new FtpCloudNioFileSystemProvider(ConfigFactory.empty, FtpAnonymousCredentials, fileSystems) val fileSystem = new FtpCloudNioFileSystem(provider, "ftp.example.com") { // Override so we don't try to connect to anything - override private[ftp] lazy val clientFactory = () => { new FTPClient() } + override private[ftp] lazy val clientFactory = () => new FTPClient() override def leaseClient = { val lease = super.leaseClient diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala index 0ba19060ab2..b4262452e35 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpCredentialsSpec.scala @@ -17,21 +17,21 @@ class FtpCredentialsSpec extends AnyFlatSpec with Matchers with MockSugar with C var loggedInWithAccount: Boolean = false var loggedInWithoutAccount: Boolean = false val client = mock[FTPClient] - client.login(anyString, anyString).responds(_ => { + client.login(anyString, anyString).responds { _ => loggedInWithoutAccount = true true - }) - client.login(anyString, anyString, anyString).responds(_ => { + } + client.login(anyString, anyString, anyString).responds { _ => loggedInWithAccount = true true - }) + } FtpAuthenticatedCredentials("user", "password", None).login(client) loggedInWithoutAccount shouldBe true loggedInWithAccount shouldBe false // reset - loggedInWithoutAccount= false + loggedInWithoutAccount = false FtpAuthenticatedCredentials("user", "password", Option("account")).login(client) loggedInWithAccount shouldBe true diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala index 86f86914103..6a186a0d5f2 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpFileSystemsSpec.scala @@ -35,8 +35,8 @@ class FtpFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Match verify(mockCreateFunction).apply(ftpCacheKey) } - class MockFtpFileSystems(conf: FtpFileSystemsConfiguration, - mockCreateFunction: FtpCacheKey => FtpCloudNioFileSystem) extends FtpFileSystems(conf) { + class MockFtpFileSystems(conf: FtpFileSystemsConfiguration, mockCreateFunction: FtpCacheKey => FtpCloudNioFileSystem) + extends FtpFileSystems(conf) { override private[ftp] def createFileSystem(key: FtpCacheKey) = mockCreateFunction(key) } } diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala index 0e86194ecbd..1796cef16b9 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/FtpUtilSpec.scala @@ -16,7 +16,7 @@ class FtpUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "autoRelease" it should "release the lease when the client fails the operation without throwing" in { - val clientPool = new FtpClientPool(1, 10.minutes, () => { new FTPClient }) + val clientPool = new FtpClientPool(1, 10.minutes, () => new FTPClient) val lease = clientPool.acquire() val action = autoRelease(IO.pure(lease)) { _ => @@ -28,7 +28,7 @@ class FtpUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "invalidate the lease when the client fails the operation by throwing" in { - val clientPool = new FtpClientPool(1, 10.minutes, () => { new FTPClient }) + val clientPool = new FtpClientPool(1, 10.minutes, () => new FTPClient) val lease = clientPool.acquire() val action = autoRelease(IO.pure(lease)) { _ => @@ -40,7 +40,7 @@ class FtpUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "release the lease when the operation succeeds" in { - val clientPool = new FtpClientPool(1, 10.minutes, () => { new FTPClient }) + val clientPool = new FtpClientPool(1, 10.minutes, () => new FTPClient) val lease = clientPool.acquire() val action = autoRelease(IO.pure(lease)) { _ => diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala index 6b70679a239..7042571b993 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseInputStreamSpec.scala @@ -22,11 +22,11 @@ class LeaseInputStreamSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat } val mockClient = mock[FTPClient] var completed: Boolean = false - mockClient.completePendingCommand().returns({ + mockClient.completePendingCommand().returns { completed = true true - }) - val clientPool = new FtpClientPool(1, 10.minutes, () => { mockClient }) + } + val clientPool = new FtpClientPool(1, 10.minutes, () => mockClient) val lease = clientPool.acquire() val leasedInputStream = new LeasedInputStream("host", "path", is, lease) diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala index ee82de6ffea..5c5bef9b9f4 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/LeaseOutputStreamSpec.scala @@ -17,11 +17,11 @@ class LeaseOutputStreamSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma val os = new TestOutputStream val mockClient = mock[FTPClient] var completed: Boolean = false - mockClient.completePendingCommand().returns({ + mockClient.completePendingCommand().returns { completed = true true - }) - val clientPool = new FtpClientPool(1, 10.minutes, () => { mockClient }) + } + val clientPool = new FtpClientPool(1, 10.minutes, () => mockClient) val lease = clientPool.acquire() val leasedOutputStream = new LeasedOutputStream("host", "path", os, lease) diff --git a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala index e3c954c8d24..42716da92cd 100644 --- a/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala +++ b/cloud-nio/cloud-nio-impl-ftp/src/test/scala/cloud/nio/impl/ftp/MockFtpFileSystem.scala @@ -22,15 +22,18 @@ trait MockFtpFileSystem extends BeforeAndAfterAll { this: Suite => connectionPort = Option(fakeFtpServer.getServerControlPort) } - override def afterAll() = { + override def afterAll() = fakeFtpServer.stop() - } - lazy val ftpFileSystemsConfiguration = FtpFileSystems.DefaultConfig.copy(connectionPort = connectionPort.getOrElse(throw new RuntimeException("Fake FTP server has not been started"))) + lazy val ftpFileSystemsConfiguration = FtpFileSystems.DefaultConfig.copy(connectionPort = + connectionPort.getOrElse(throw new RuntimeException("Fake FTP server has not been started")) + ) lazy val ftpFileSystems = new FtpFileSystems(ftpFileSystemsConfiguration) // Do not call this before starting the server - lazy val mockProvider = { - new FtpCloudNioFileSystemProvider(ConfigFactory.empty, FtpAuthenticatedCredentials("test_user", "test_password", None), ftpFileSystems) - } + lazy val mockProvider = + new FtpCloudNioFileSystemProvider(ConfigFactory.empty, + FtpAuthenticatedCredentials("test_user", "test_password", None), + ftpFileSystems + ) } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala index e55031f5caa..eb0ae65df29 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioBackoff.scala @@ -6,8 +6,10 @@ import com.google.api.client.util.ExponentialBackOff import scala.concurrent.duration.{Duration, FiniteDuration} trait CloudNioBackoff { + /** Next interval in millis */ def backoffMillis: Long + /** Get the next instance of backoff. This should be called after every call to backoffMillis */ def next: CloudNioBackoff } @@ -16,8 +18,8 @@ object CloudNioBackoff { private[spi] def newExponentialBackOff(initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, - randomizationFactor: Double, - ): ExponentialBackOff = { + randomizationFactor: Double + ): ExponentialBackOff = new ExponentialBackOff.Builder() .setInitialIntervalMillis(initialInterval.toMillis.toInt) .setMaxIntervalMillis(maxInterval.toMillis.toInt) @@ -25,7 +27,6 @@ object CloudNioBackoff { .setRandomizationFactor(randomizationFactor) .setMaxElapsedTimeMillis(Int.MaxValue) .build() - } } object CloudNioInitialGapBackoff { @@ -33,24 +34,25 @@ object CloudNioInitialGapBackoff { initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, - randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR, - ): CloudNioInitialGapBackoff = { + randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR + ): CloudNioInitialGapBackoff = new CloudNioInitialGapBackoff( initialGap, newExponentialBackOff( initialInterval = initialInterval, maxInterval = maxInterval, multiplier = multiplier, - randomizationFactor = randomizationFactor, + randomizationFactor = randomizationFactor ) ) - } } -case class CloudNioInitialGapBackoff(initialGapMillis: FiniteDuration, googleBackoff: ExponentialBackOff) extends CloudNioBackoff { +case class CloudNioInitialGapBackoff(initialGapMillis: FiniteDuration, googleBackoff: ExponentialBackOff) + extends CloudNioBackoff { assert(initialGapMillis.compareTo(Duration.Zero) != 0, "Initial gap cannot be null, use SimpleBackoff instead.") override val backoffMillis: Long = initialGapMillis.toMillis + /** Switch to a SimpleExponentialBackoff after the initial gap has been used */ override def next = new CloudNioSimpleExponentialBackoff(googleBackoff) } @@ -59,21 +61,21 @@ object CloudNioSimpleExponentialBackoff { def apply(initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, - randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR, - ): CloudNioSimpleExponentialBackoff = { + randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR + ): CloudNioSimpleExponentialBackoff = new CloudNioSimpleExponentialBackoff( newExponentialBackOff( initialInterval = initialInterval, maxInterval = maxInterval, multiplier = multiplier, - randomizationFactor = randomizationFactor, + randomizationFactor = randomizationFactor ) ) - } } case class CloudNioSimpleExponentialBackoff(googleBackoff: ExponentialBackOff) extends CloudNioBackoff { override def backoffMillis: Long = googleBackoff.nextBackOffMillis() + /** google ExponentialBackOff is mutable so we can keep returning the same instance */ override def next: CloudNioBackoff = this } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala index 909c1fc54db..25d0f34e7dc 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioDirectoryStream.scala @@ -13,24 +13,19 @@ class CloudNioDirectoryStream( override def iterator(): java.util.Iterator[Path] = pathStream().filterNot(_ == prefix).iterator.asJava - private[this] def pathStream(markerOption: Option[String] = None): LazyList[Path] = { + private[this] def pathStream(markerOption: Option[String] = None): LazyList[Path] = listNext(markerOption) match { case CloudNioFileList(keys, Some(marker)) => keys.to(LazyList).map(toPath) ++ pathStream(Option(marker)) case CloudNioFileList(keys, None) => keys.to(LazyList).map(toPath) } - } - private[this] def toPath(key: String): Path = { + private[this] def toPath(key: String): Path = prefix.getFileSystem.getPath("/" + key) - } - private[this] def listNext(markerOption: Option[String]): CloudNioFileList = { - retry.from( - () => fileProvider.listObjects(prefix.cloudHost, prefix.cloudPath, markerOption) - ) - } + private[this] def listNext(markerOption: Option[String]): CloudNioFileList = + retry.from(() => fileProvider.listObjects(prefix.cloudHost, prefix.cloudPath, markerOption)) override def close(): Unit = {} diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala index 7cc34400676..852951784ac 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileAttributeView.scala @@ -11,21 +11,17 @@ final case class CloudNioFileAttributeView( ) extends BasicFileAttributeView { override def name(): String = CloudNioFileAttributeView.Name - override def readAttributes(): CloudNioFileAttributes = { + override def readAttributes(): CloudNioFileAttributes = if (isDirectory) { CloudNioDirectoryAttributes(cloudNioPath) } else { retry - .from( - () => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + .from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) .getOrElse(throw new FileNotFoundException(cloudNioPath.uriAsString)) } - } - override def setTimes(lastModifiedTime: FileTime, lastAccessTime: FileTime, createTime: FileTime): Unit = { + override def setTimes(lastModifiedTime: FileTime, lastAccessTime: FileTime, createTime: FileTime): Unit = throw new UnsupportedOperationException - } } object CloudNioFileAttributeView { diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala index 8b93419f350..43384330388 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystem.scala @@ -20,49 +20,40 @@ class CloudNioFileSystem(override val provider: CloudNioFileSystemProvider, val // do nothing currently. } - override def isOpen: Boolean = { + override def isOpen: Boolean = true - } - override def isReadOnly: Boolean = { + override def isReadOnly: Boolean = false - } - override def getSeparator: String = { + override def getSeparator: String = CloudNioFileSystem.Separator - } - override def getRootDirectories: java.lang.Iterable[Path] = { + override def getRootDirectories: java.lang.Iterable[Path] = Set[Path](getPath(UnixPath.Root)).asJava - } - override def getFileStores: java.lang.Iterable[FileStore] = { + override def getFileStores: java.lang.Iterable[FileStore] = Set.empty[FileStore].asJava - } - override def getPathMatcher(syntaxAndPattern: String): PathMatcher = { + override def getPathMatcher(syntaxAndPattern: String): PathMatcher = FileSystems.getDefault.getPathMatcher(syntaxAndPattern) - } - override def getUserPrincipalLookupService: UserPrincipalLookupService = { + override def getUserPrincipalLookupService: UserPrincipalLookupService = throw new UnsupportedOperationException - } - override def newWatchService(): WatchService = { + override def newWatchService(): WatchService = throw new UnsupportedOperationException - } - override def supportedFileAttributeViews(): java.util.Set[String] = { + override def supportedFileAttributeViews(): java.util.Set[String] = Set("basic", CloudNioFileAttributeView.Name).asJava - } def canEqual(other: Any): Boolean = other.isInstanceOf[CloudNioFileSystem] override def equals(other: Any): Boolean = other match { case that: CloudNioFileSystem => (that canEqual this) && - provider == that.provider && - host == that.host + provider == that.provider && + host == that.host case _ => false } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala index 9b79e308afd..c78d30d2dce 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioFileSystemProvider.scala @@ -3,7 +3,7 @@ package cloud.nio.spi import java.net.URI import java.nio.channels.SeekableByteChannel import java.nio.file._ -import java.nio.file.attribute.{BasicFileAttributeView, BasicFileAttributes, FileAttribute, FileAttributeView} +import java.nio.file.attribute.{BasicFileAttributes, BasicFileAttributeView, FileAttribute, FileAttributeView} import java.nio.file.spi.FileSystemProvider import com.typesafe.config.{Config, ConfigFactory} @@ -64,9 +64,8 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { newCloudNioFileSystem(uri.toString, config) } - override def getPath(uri: URI): CloudNioPath = { + override def getPath(uri: URI): CloudNioPath = getFileSystem(uri).getPath(uri.getPath) - } override def newByteChannel( path: Path, @@ -75,7 +74,7 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { ): SeekableByteChannel = { val cloudNioPath = CloudNioPath.checkPath(path) - for (opt <- options.asScala) { + for (opt <- options.asScala) opt match { case StandardOpenOption.READ | StandardOpenOption.WRITE | StandardOpenOption.SPARSE | StandardOpenOption.TRUNCATE_EXISTING | StandardOpenOption.CREATE | StandardOpenOption.CREATE_NEW => @@ -84,7 +83,6 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { StandardOpenOption.SYNC => throw new UnsupportedOperationException(opt.toString) } - } if (options.contains(StandardOpenOption.READ) && options.contains(StandardOpenOption.WRITE)) { throw new UnsupportedOperationException("Cannot open a READ+WRITE channel") @@ -95,70 +93,64 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { } } - protected def cloudNioReadChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioReadChannel = new CloudNioReadChannel(fileProvider, retry, cloudNioPath) - protected def cloudNioWriteChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioWriteChannel = new CloudNioWriteChannel(fileProvider, retry, cloudNioPath) + protected def cloudNioReadChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioReadChannel = + new CloudNioReadChannel(fileProvider, retry, cloudNioPath) + protected def cloudNioWriteChannel(retry: CloudNioRetry, cloudNioPath: CloudNioPath): CloudNioWriteChannel = + new CloudNioWriteChannel(fileProvider, retry, cloudNioPath) - override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = retry.from(() => { + override def createDirectory(dir: Path, attrs: FileAttribute[_]*): Unit = retry.from { () => val cloudNioPath = CloudNioPath.checkPath(dir) fileProvider.createDirectory(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - }) + } override def deleteIfExists(path: Path): Boolean = { val cloudNioPath = CloudNioPath.checkPath(path) if (checkDirectoryExists(cloudNioPath)) { - val hasObjects = retry.from( - () => fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + val hasObjects = retry.from(() => fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) if (hasObjects) { throw new UnsupportedOperationException("Can not delete a non-empty directory") } else { true } } else { - retry.from( - () => fileProvider.deleteIfExists(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + retry.from(() => fileProvider.deleteIfExists(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) } } - override def delete(path: Path): Unit = { + override def delete(path: Path): Unit = if (!deleteIfExists(path)) { val cloudNioPath = CloudNioPath.checkPath(path) throw new NoSuchFileException(cloudNioPath.uriAsString) } - } override def copy(source: Path, target: Path, options: CopyOption*): Unit = { val sourceCloudNioPath = CloudNioPath.checkPath(source) val targetCloudNioPath = CloudNioPath.checkPath(target) if (sourceCloudNioPath != targetCloudNioPath) { - retry.from( - () => - fileProvider.copy( - sourceCloudNioPath.cloudHost, - sourceCloudNioPath.cloudPath, - targetCloudNioPath.cloudHost, - targetCloudNioPath.cloudPath - ) + retry.from(() => + fileProvider.copy( + sourceCloudNioPath.cloudHost, + sourceCloudNioPath.cloudPath, + targetCloudNioPath.cloudHost, + targetCloudNioPath.cloudPath + ) ) } } override def move(source: Path, target: Path, options: CopyOption*): Unit = { - for (option <- options) { + for (option <- options) if (option == StandardCopyOption.ATOMIC_MOVE) throw new AtomicMoveNotSupportedException(null, null, "Atomic move unsupported") - } copy(source, target, options: _*) delete(source) () } - override def isSameFile(path: Path, path2: Path): Boolean = { + override def isSameFile(path: Path, path2: Path): Boolean = CloudNioPath.checkPath(path).equals(CloudNioPath.checkPath(path2)) - } override def isHidden(path: Path): Boolean = { CloudNioPath.checkPath(path) @@ -174,8 +166,8 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { val cloudNioPath = CloudNioPath.checkPath(path) - val exists = checkDirectoryExists(cloudNioPath) || retry.from( - () => fileProvider.existsPath(cloudNioPath.cloudHost, cloudNioPath.cloudPath) + val exists = checkDirectoryExists(cloudNioPath) || retry.from(() => + fileProvider.existsPath(cloudNioPath.cloudHost, cloudNioPath.cloudPath) ) if (!exists) { @@ -183,12 +175,11 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { } } - def checkDirectoryExists(cloudNioPath: CloudNioPath): Boolean = { + def checkDirectoryExists(cloudNioPath: CloudNioPath): Boolean = // Anything that "seems" like a directory exists. Otherwise see if the path with a "/" contains files on the cloud. - (usePseudoDirectories && cloudNioPath.seemsLikeDirectory) || retry.from( - () => fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath + "/") + (usePseudoDirectories && cloudNioPath.seemsLikeDirectory) || retry.from(() => + fileProvider.existsPaths(cloudNioPath.cloudHost, cloudNioPath.cloudPath + "/") ) - } override def getFileAttributeView[V <: FileAttributeView]( path: Path, @@ -205,9 +196,8 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { CloudNioFileAttributeView(fileProvider, retry, cloudNioPath, isDirectory).asInstanceOf[V] } - override def readAttributes(path: Path, attributes: String, options: LinkOption*): java.util.Map[String, AnyRef] = { + override def readAttributes(path: Path, attributes: String, options: LinkOption*): java.util.Map[String, AnyRef] = throw new UnsupportedOperationException - } override def readAttributes[A <: BasicFileAttributes]( path: Path, @@ -224,9 +214,7 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { CloudNioDirectoryAttributes(cloudNioPath).asInstanceOf[A] } else { retry - .from( - () => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) + .from(() => fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) .map(_.asInstanceOf[A]) .getOrElse(throw new NoSuchFileException(cloudNioPath.uriAsString)) } @@ -237,16 +225,15 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { new CloudNioDirectoryStream(fileProvider, retry, cloudNioPath, filter) } - override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = { + override def setAttribute(path: Path, attribute: String, value: scala.Any, options: LinkOption*): Unit = throw new UnsupportedOperationException - } def canEqual(other: Any): Boolean = other.isInstanceOf[CloudNioFileSystemProvider] override def equals(other: Any): Boolean = other match { case that: CloudNioFileSystemProvider => (that canEqual this) && - config == that.config + config == that.config case _ => false } @@ -258,7 +245,6 @@ abstract class CloudNioFileSystemProvider extends FileSystemProvider { object CloudNioFileSystemProvider { - def defaultConfig(scheme: String): Config = { + def defaultConfig(scheme: String): Config = ConfigFactory.load.getOrElse(s"cloud.nio.default.$scheme", ConfigFactory.empty) - } } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala index ec0c701a9b9..530ba5526ef 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioPath.scala @@ -9,12 +9,11 @@ import scala.jdk.CollectionConverters._ object CloudNioPath { - def checkPath(path: Path): CloudNioPath = { + def checkPath(path: Path): CloudNioPath = path match { case cloudNioPath: CloudNioPath => cloudNioPath - case _ => throw new ProviderMismatchException(s"Not a CloudNioPath: $path") + case _ => throw new ProviderMismatchException(s"Not a CloudNioPath: $path") } - } } class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: UnixPath) extends Path { @@ -65,13 +64,12 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un /** * If is relative, returns just the normalized path. If is absolute, return the host + the absolute path. */ - def relativeDependentPath: String = { + def relativeDependentPath: String = if (unixPath.isAbsolute) { cloudHost + "/" + unixPath.toString.stripPrefix("/") } else { unixPath.normalize().toString } - } /** * Returns true if the path probably represents a directory, but won't be known until contacting the host. @@ -84,30 +82,25 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un override def isAbsolute: Boolean = unixPath.isAbsolute - override def getRoot: CloudNioPath = { + override def getRoot: CloudNioPath = unixPath.getRoot.map(newPath).orNull - } - override def getFileName: CloudNioPath = { + override def getFileName: CloudNioPath = unixPath.getFileName.map(newPath).orNull - } - override def getParent: CloudNioPath = { + override def getParent: CloudNioPath = unixPath.getParent.map(newPath).orNull - } override def getNameCount: Int = unixPath.getNameCount - override def getName(index: Int): CloudNioPath = { + override def getName(index: Int): CloudNioPath = unixPath.getName(index).map(newPath).getOrElse(throw new IllegalArgumentException(s"Bad index $index")) - } - override def subpath(beginIndex: Int, endIndex: Int): CloudNioPath = { + override def subpath(beginIndex: Int, endIndex: Int): CloudNioPath = unixPath .subPath(beginIndex, endIndex) .map(newPath) .getOrElse(throw new IllegalArgumentException(s"Bad range $beginIndex-$endIndex")) - } override def startsWith(other: Path): Boolean = { if (!other.isInstanceOf[CloudNioPath]) { @@ -122,9 +115,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un unixPath.startsWith(that.unixPath) } - override def startsWith(other: String): Boolean = { + override def startsWith(other: String): Boolean = unixPath.startsWith(UnixPath.getPath(other)) - } override def endsWith(other: Path): Boolean = { if (!other.isInstanceOf[CloudNioPath]) { @@ -138,9 +130,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un unixPath.endsWith(that.unixPath) } - override def endsWith(other: String): Boolean = { + override def endsWith(other: String): Boolean = unixPath.endsWith(UnixPath.getPath(other)) - } override def normalize(): CloudNioPath = newPath(unixPath.normalize()) @@ -150,9 +141,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un newPath(unixPath.resolve(that.unixPath)) } - override def resolve(other: String): CloudNioPath = { + override def resolve(other: String): CloudNioPath = newPath(unixPath.resolve(UnixPath.getPath(other))) - } override def resolveSibling(other: Path): CloudNioPath = { val that = CloudNioPath.checkPath(other) @@ -160,9 +150,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un newPath(unixPath.resolveSibling(that.unixPath)) } - override def resolveSibling(other: String): CloudNioPath = { + override def resolveSibling(other: String): CloudNioPath = newPath(unixPath.resolveSibling(UnixPath.getPath(other))) - } override def relativize(other: Path): CloudNioPath = { val that = CloudNioPath.checkPath(other) @@ -170,9 +159,8 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un newPath(unixPath.relativize(that.unixPath)) } - override def toAbsolutePath: CloudNioPath = { + override def toAbsolutePath: CloudNioPath = newPath(unixPath.toAbsolutePath) - } override def toRealPath(options: LinkOption*): CloudNioPath = toAbsolutePath @@ -187,13 +175,12 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un modifiers: WatchEvent.Modifier* ): WatchKey = throw new UnsupportedOperationException - override def iterator(): java.util.Iterator[Path] = { - if (unixPath.isEmpty || unixPath.isRoot) { + override def iterator(): java.util.Iterator[Path] = + if (unixPath.izEmpty || unixPath.isRoot) { java.util.Collections.emptyIterator() } else { unixPath.split().to(LazyList).map(part => newPath(UnixPath.getPath(part)).asInstanceOf[Path]).iterator.asJava } - } override def compareTo(other: Path): Int = { if (other.isInstanceOf[CloudNioPath]) { @@ -209,22 +196,19 @@ class CloudNioPath(filesystem: CloudNioFileSystem, private[spi] val unixPath: Un unixPath.compareTo(that.unixPath) } - override def equals(obj: scala.Any): Boolean = { + override def equals(obj: scala.Any): Boolean = (this eq obj.asInstanceOf[AnyRef]) || - obj.isInstanceOf[CloudNioPath] && - obj.asInstanceOf[CloudNioPath].cloudHost.equals(cloudHost) && - obj.asInstanceOf[CloudNioPath].unixPath.equals(unixPath) - } + obj.isInstanceOf[CloudNioPath] && + obj.asInstanceOf[CloudNioPath].cloudHost.equals(cloudHost) && + obj.asInstanceOf[CloudNioPath].unixPath.equals(unixPath) - override def hashCode(): Int = { + override def hashCode(): Int = Objects.hash(cloudHost, unixPath) - } - protected def newPath(unixPath: UnixPath): CloudNioPath = { + protected def newPath(unixPath: UnixPath): CloudNioPath = if (this.unixPath == unixPath) { this } else { new CloudNioPath(filesystem, unixPath) } - } } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala index 1e6020307fa..b8260b9a599 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioReadChannel.scala @@ -2,35 +2,28 @@ package cloud.nio.spi import java.io.FileNotFoundException import java.nio.ByteBuffer -import java.nio.channels.{ - ClosedChannelException, - NonWritableChannelException, - ReadableByteChannel, - SeekableByteChannel -} +import java.nio.channels.{ClosedChannelException, NonWritableChannelException, ReadableByteChannel, SeekableByteChannel} class CloudNioReadChannel(fileProvider: CloudNioFileProvider, retry: CloudNioRetry, cloudNioPath: CloudNioPath) - extends SeekableByteChannel { + extends SeekableByteChannel { private var internalPosition: Long = 0 private var channel: ReadableByteChannel = channelPosition(0) override def read(dst: ByteBuffer): Int = { var resetConnection = false - val count = retry.from( - () => { - try { - if (resetConnection) { - if (channel.isOpen) channel.close() - channel = fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, internalPosition) - } - channel.read(dst) - } catch { - case exception: Exception => - resetConnection = true - throw exception + val count = retry.from { () => + try { + if (resetConnection) { + if (channel.isOpen) channel.close() + channel = fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, internalPosition) } + channel.read(dst) + } catch { + case exception: Exception => + resetConnection = true + throw exception } - ) + } if (count > 0) internalPosition += count count @@ -50,25 +43,19 @@ class CloudNioReadChannel(fileProvider: CloudNioFileProvider, retry: CloudNioRet this } - private def channelPosition(newPosition: Long): ReadableByteChannel = { - retry.from( - () => fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, newPosition) - ) - } + private def channelPosition(newPosition: Long): ReadableByteChannel = + retry.from(() => fileProvider.read(cloudNioPath.cloudHost, cloudNioPath.cloudPath, newPosition)) - override def size(): Long = { + override def size(): Long = retry - .from( - () => fileSize - ) + .from(() => fileSize) .getOrElse(throw new FileNotFoundException(cloudNioPath.uriAsString)) - } override def truncate(size: Long): SeekableByteChannel = throw new NonWritableChannelException override def isOpen: Boolean = channel.isOpen override def close(): Unit = channel.close() - + protected def fileSize = fileProvider.fileAttributes(cloudNioPath.cloudHost, cloudNioPath.cloudPath).map(_.size()) } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala index 9a4028ed859..40c8cba7377 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioRetry.scala @@ -24,7 +24,7 @@ class CloudNioRetry(config: Config) { val delay = backoff.backoffMillis f() match { - case Success(ret) => ret + case Success(ret) => ret case Failure(exception: Exception) if isFatal(exception) => throw exception case Failure(exception: Exception) if !isFatal(exception) => val retriesLeft = if (isTransient(exception)) maxRetries else maxRetries map { _ - 1 } @@ -38,11 +38,13 @@ class CloudNioRetry(config: Config) { } } - def from[A](f: () => A, maxRetries: Option[Int] = Option(defaultMaxRetries), backoff: CloudNioBackoff = defaultBackOff): A = { + def from[A](f: () => A, + maxRetries: Option[Int] = Option(defaultMaxRetries), + backoff: CloudNioBackoff = defaultBackOff + ): A = fromTry[A]( () => Try(f()), maxRetries, backoff ) - } } diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala index 15a9d97213b..232cc25297b 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/CloudNioWriteChannel.scala @@ -6,11 +6,8 @@ import java.nio.channels._ class CloudNioWriteChannel(fileProvider: CloudNioFileProvider, retry: CloudNioRetry, cloudNioPath: CloudNioPath) extends SeekableByteChannel { private var internalPosition: Long = 0 - private val channel: WritableByteChannel = { - retry.from( - () => fileProvider.write(cloudNioPath.cloudHost, cloudNioPath.cloudPath) - ) - } + private val channel: WritableByteChannel = + retry.from(() => fileProvider.write(cloudNioPath.cloudHost, cloudNioPath.cloudPath)) override def read(dst: ByteBuffer): Int = throw new NonReadableChannelException diff --git a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala index fb3a2bf8f50..41d742a4e66 100644 --- a/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala +++ b/cloud-nio/cloud-nio-spi/src/main/scala/cloud/nio/spi/UnixPath.scala @@ -24,7 +24,7 @@ private[spi] object UnixPath { private def hasTrailingSeparator(path: String): Boolean = !path.isEmpty && path.charAt(path.length - 1) == Separator - def getPath(path: String): UnixPath = { + def getPath(path: String): UnixPath = if (path.isEmpty) { EmptyPath } else if (isRoot(path)) { @@ -32,7 +32,6 @@ private[spi] object UnixPath { } else { UnixPath(path) } - } def getPath(first: String, more: String*): UnixPath = { if (more.isEmpty) { @@ -40,7 +39,7 @@ private[spi] object UnixPath { } val builder = new StringBuilder(first) - for ((part, index) <- more.view.zipWithIndex) { + for ((part, index) <- more.view.zipWithIndex) if (part.isEmpty) { // do nothing } else if (isAbsolute(part)) { @@ -55,7 +54,6 @@ private[spi] object UnixPath { builder.append(Separator) builder.append(part) } - } UnixPath(builder.toString) } @@ -69,18 +67,21 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { def isAbsolute: Boolean = UnixPath.isAbsolute(path) - def isEmpty: Boolean = path.isEmpty + // Named this way because isEmpty is a name collision new in 17. + // The initial compile error is that it needs an override. + // Adding the override results in a second error saying it overrides nothing! + // So, we just renamed it. + def izEmpty: Boolean = path.isEmpty def hasTrailingSeparator: Boolean = UnixPath.hasTrailingSeparator(path) - def seemsLikeDirectory(): Boolean = { + def seemsLikeDirectory(): Boolean = path.isEmpty || - hasTrailingSeparator || - path.endsWith(".") && (length == 1 || path.charAt(length - 2) == UnixPath.Separator) || - path.endsWith("..") && (length == 2 || path.charAt(length - 3) == UnixPath.Separator) - } + hasTrailingSeparator || + path.endsWith(".") && (length == 1 || path.charAt(length - 2) == UnixPath.Separator) || + path.endsWith("..") && (length == 2 || path.charAt(length - 3) == UnixPath.Separator) - def getFileName: Option[UnixPath] = { + def getFileName: Option[UnixPath] = if (path.isEmpty || isRoot) { None } else { @@ -90,7 +91,6 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { Some(UnixPath(parts.last)) } } - } def getParent: Option[UnixPath] = { if (path.isEmpty || isRoot) { @@ -103,7 +103,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { else path.lastIndexOf(UnixPath.Separator.toInt) index match { - case -1 => if (isAbsolute) Some(UnixPath.RootPath) else None + case -1 => if (isAbsolute) Some(UnixPath.RootPath) else None case pos => Some(UnixPath(path.substring(0, pos + 1))) } } @@ -122,7 +122,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { Try(UnixPath(parts.slice(beginIndex, endIndex).mkString(UnixPath.Separator.toString))) } - def getNameCount: Int = { + def getNameCount: Int = if (path.isEmpty) { 1 } else if (isRoot) { @@ -130,7 +130,6 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } else { parts.length } - } def getName(index: Int): Try[UnixPath] = { if (path.isEmpty) { @@ -144,7 +143,7 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { Success(UnixPath(parts(2))) } - def resolve(other: UnixPath): UnixPath = { + def resolve(other: UnixPath): UnixPath = if (other.path.isEmpty) { this } else if (other.isAbsolute) { @@ -154,15 +153,13 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } else { new UnixPath(path + UnixPath.Separator.toString + other.path) } - } - def resolveSibling(other: UnixPath): UnixPath = { + def resolveSibling(other: UnixPath): UnixPath = getParent match { case Some(parent: UnixPath) => parent.resolve(other) case None => other } - } def relativize(other: UnixPath): UnixPath = { if (path.isEmpty) { @@ -247,21 +244,18 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { def splitReverse(): Iterator[String] = parts.reverseIterator - def removeBeginningSeparator(): UnixPath = { + def removeBeginningSeparator(): UnixPath = if (isAbsolute) new UnixPath(path.substring(1)) else this - } - def addTrailingSeparator(): UnixPath = { + def addTrailingSeparator(): UnixPath = if (hasTrailingSeparator) this else new UnixPath(path + UnixPath.Separator) - } - def removeTrailingSeparator(): UnixPath = { + def removeTrailingSeparator(): UnixPath = if (!isRoot && hasTrailingSeparator) { new UnixPath(path.substring(0, length - 1)) } else { this } - } def startsWith(other: UnixPath): Boolean = { val me = removeTrailingSeparator() @@ -279,11 +273,10 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } def startsWith(left: Iterator[String], right: Iterator[String]): Boolean = { - while (right.hasNext) { + while (right.hasNext) if (!left.hasNext || right.next() != left.next()) { return false } - } true } @@ -310,9 +303,8 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { if (isAbsolute) Success(this) else Success(currentWorkingDirectory.resolve(this)) } - def toAbsolutePath: UnixPath = { + def toAbsolutePath: UnixPath = if (isAbsolute) this else UnixPath.RootPath.resolve(this) - } def compareTo(other: UnixPath): Int = { val me = parts.toList @@ -327,29 +319,24 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { } } - override def equals(obj: scala.Any): Boolean = { + override def equals(obj: scala.Any): Boolean = (this eq obj.asInstanceOf[AnyRef]) || { obj.isInstanceOf[UnixPath] && obj.asInstanceOf[UnixPath].path.equals(path) } - } - override def length(): Int = { + override def length(): Int = path.length - } - override def charAt(index: Int): Char = { + override def charAt(index: Int): Char = path.charAt(index) - } - override def subSequence(start: Int, end: Int): CharSequence = { + override def subSequence(start: Int, end: Int): CharSequence = path.subSequence(start, end) - } - override def toString: String = { + override def toString: String = path - } - def initParts(): Array[String] = { + def initParts(): Array[String] = if (path.isEmpty) { Array.empty[String] } else { @@ -359,5 +346,4 @@ final private[spi] case class UnixPath(path: String) extends CharSequence { path.split(UnixPath.Separator) } } - } } diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala index 04c9d4cdd3f..bbd1cbaf4db 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/ChannelUtil.scala @@ -26,13 +26,12 @@ object ChannelUtil { def pipedStreamWriter(threadName: String)(consumer: InputStream => Unit): WritableByteChannel = { val pipe = Pipe.open() var threadResult: Option[Try[Unit]] = None - val runnable: Runnable = () => { + val runnable: Runnable = () => threadResult = Option( Try( consumer(Channels.newInputStream(pipe.source)) ) ) - } val thread = new Thread(runnable, threadName) thread.setDaemon(true) thread.start() diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala index 162d47d08db..e15cb574031 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioFiles.scala @@ -12,13 +12,12 @@ object CloudNioFiles { /** * Lists all files under a path. */ - def listRegularFiles(path: Path): Iterator[Path] = { + def listRegularFiles(path: Path): Iterator[Path] = Files .walk(path, Int.MaxValue) .iterator .asScala .filter(Files.isRegularFile(_)) - } /** * Returns an iterator of all regular files under sourcePath mapped relatively to targetPath. diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala index 7448b272426..66ab48100c6 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/CloudNioPaths.scala @@ -21,15 +21,14 @@ object CloudNioPaths { * @see [[cloud.nio.util.CloudNioPaths#showAbsolute(java.nio.file.Path)]] * @see [[cloud.nio.spi.CloudNioPath#uriAsString()]] */ - def get(filePath: String): Path = { - try { + def get(filePath: String): Path = + try // TODO: softer parsing using Guava UrlEscapers. May also be better to list the providers ourselves if possible. Paths.get(new URI(filePath)) - } catch { - case _: URISyntaxException => Paths.get(filePath) + catch { + case _: URISyntaxException => Paths.get(filePath) case iae: IllegalArgumentException if iae.getMessage == "Missing scheme" => Paths.get(filePath) } - } /** * Return a path in a way reciprocal with [[cloud.nio.util.CloudNioPaths#get]]. @@ -38,12 +37,11 @@ object CloudNioPaths { * @see [[cloud.nio.util.CloudNioPaths#showRelative(java.nio.file.Path)]] * @see [[cloud.nio.spi.CloudNioPath#uriAsString()]] */ - def showAbsolute(path: Path): String = { + def showAbsolute(path: Path): String = path match { case cloudNioPath: CloudNioPath => cloudNioPath.uriAsString - case _ => path.toAbsolutePath.toString + case _ => path.toAbsolutePath.toString } - } /** * When the path is relative returns a relative path in a way reciprocal with resolve. @@ -53,11 +51,10 @@ object CloudNioPaths { * @see [[java.nio.file.Path#resolve(java.nio.file.Path)]] * @see [[cloud.nio.spi.CloudNioPath#uriAsString()]] */ - def showRelative(path: Path): String = { + def showRelative(path: Path): String = path match { case cloudNioPath: CloudNioPath => cloudNioPath.relativeDependentPath - case _ if !path.isAbsolute => path.normalize().toString - case _ => path.getRoot.relativize(path).normalize().toString + case _ if !path.isAbsolute => path.normalize().toString + case _ => path.getRoot.relativize(path).normalize().toString } - } } diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala index 0ff44ce2dd0..f8c7921d0d8 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/IoUtil.scala @@ -11,24 +11,21 @@ object IoUtil { type ACC = IO[Either[NonEmptyList[Exception], A]] - def attemptHead(headIo: IO[A]): ACC = { + def attemptHead(headIo: IO[A]): ACC = attemptIo(NonEmptyList.one)(headIo) - } - def attemptAcc(accIo: ACC, nextIo: IO[A]): ACC = { + def attemptAcc(accIo: ACC, nextIo: IO[A]): ACC = accIo flatMap { - case Right(previousSuccess) => IO.pure(Right(previousSuccess)) + case Right(previousSuccess) => IO.pure(Right(previousSuccess)) case Left(previousExceptions) => attemptIo(_ :: previousExceptions)(nextIo) } - } - def attemptIo(f: Exception => NonEmptyList[Exception])(io: IO[A]): ACC = { + def attemptIo(f: Exception => NonEmptyList[Exception])(io: IO[A]): ACC = io.attempt flatMap { - case Right(success) => IO.pure(Right(success)) + case Right(success) => IO.pure(Right(success)) case Left(exception: Exception) => IO.pure(Left(f(exception))) - case Left(throwable) => throw throwable + case Left(throwable) => throw throwable } - } val res: ACC = tries.tail.foldLeft(attemptHead(tries.head))(attemptAcc) diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala index 326a1821d84..f43ad24cb1f 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/TryWithResource.scala @@ -20,17 +20,17 @@ object TryWithResource { case x: Throwable => t = Option(x) throw x - } finally { + } finally resource foreach { r => - try { + try r.close() - } catch { - case y: Throwable => t match { - case Some(_t) => _t.addSuppressed(y) - case None => throw y - } + catch { + case y: Throwable => + t match { + case Some(_t) => _t.addSuppressed(y) + case None => throw y + } } } - } } } diff --git a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala index 019dbe81caf..c11651879fa 100644 --- a/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala +++ b/cloud-nio/cloud-nio-util/src/main/scala/cloud/nio/util/VersionUtil.scala @@ -35,19 +35,17 @@ object VersionUtil { * @param default What to return when the version cannot be found. The parameter passed is the `projectName`. * @return The version from the conf or the default */ - def getVersion(projectName: String, default: String => String = defaultMessage): String = { + def getVersion(projectName: String, default: String => String = defaultMessage): String = ConfigFactory .load(versionConf(projectName)) .as[Option[String]](versionProperty(projectName)) .getOrElse(default(projectName)) - } /** * Instead of returning a version, states that the version conf will be generated by sbt. */ - def defaultMessage(projectName: String): String = { + def defaultMessage(projectName: String): String = s"${versionConf(projectName)}-to-be-generated-by-sbt" - } /** * A regex compatible with the dependency constants in project/Dependencies.scala. @@ -62,7 +60,7 @@ object VersionUtil { * @return The dependency version from project/Dependencies.scala * @throws RuntimeException If the dependency cannot be found */ - def sbtDependencyVersion(dependencyName: String)(projectName: String): String = { + def sbtDependencyVersion(dependencyName: String)(projectName: String): String = try { val dependencies = Paths.get("project/Dependencies.scala").toAbsolutePath val lines = Files.readAllLines(dependencies).asScala @@ -79,6 +77,5 @@ object VersionUtil { e ) } - } } diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala index 1aca98c9efd..df2cef13bde 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/AwsConfiguration.scala @@ -46,16 +46,16 @@ import software.amazon.awssdk.regions.Region final case class AwsConfiguration private (applicationName: String, authsByName: Map[String, AwsAuthMode], - strRegion: Option[String]) { + strRegion: Option[String] +) { - def auth(name: String): ErrorOr[AwsAuthMode] = { + def auth(name: String): ErrorOr[AwsAuthMode] = authsByName.get(name) match { case None => val knownAuthNames = authsByName.keys.mkString(", ") s"`aws` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".invalidNel case Some(a) => a.validNel } - } def region: Option[Region] = strRegion.map(Region.of) } @@ -77,7 +77,7 @@ object AwsConfiguration { val awsConfig = config.getConfig("aws") - val appName = validate { awsConfig.as[String]("application-name") } + val appName = validate(awsConfig.as[String]("application-name")) val region: Option[String] = awsConfig.getAs[String]("region") @@ -88,13 +88,16 @@ object AwsConfiguration { (authConfig.getAs[String]("access-key"), authConfig.getAs[String]("secret-key")) match { case (Some(accessKey), Some(secretKey)) => CustomKeyMode(name, accessKey, secretKey, region) - case _ => throw new ConfigException.Generic(s"""Access key and/or secret """ + - s"""key missing for service account "$name". See reference.conf under the aws.auth, """ + - s"""custom key section for details of required configuration.""") + case _ => + throw new ConfigException.Generic( + s"""Access key and/or secret """ + + s"""key missing for service account "$name". See reference.conf under the aws.auth, """ + + s"""custom key section for details of required configuration.""" + ) } } - def defaultAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate { + def defaultAuth(authConfig: Config, name: String, region: Option[String]): ErrorOr[AwsAuthMode] = validate { DefaultMode(name, region) } diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala index b8c351c4185..7335441ea80 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/auth/AwsAuthMode.scala @@ -52,9 +52,8 @@ sealed trait AwsAuthMode { /** * Validate the auth mode against provided options */ - def validate(options: OptionLookup): Unit = { + def validate(options: OptionLookup): Unit = () - } /** * The name of the auth mode @@ -72,27 +71,30 @@ sealed trait AwsAuthMode { * All traits in this file are sealed, all classes final, meaning things * like Mockito or other java/scala overrides cannot work. */ - private[auth] var credentialValidation: (AwsCredentialsProvider, Option[String]) => Unit = - (provider: AwsCredentialsProvider, region: Option[String]) => { - val builder = StsClient.builder - - //If the region argument exists in config, set it in the builder. - //Otherwise it is left unset and the AwsCredentialsProvider will be responsible for sourcing a region - region.map(Region.of).foreach(builder.region) - - // make an essentially no-op call just to assure ourselves the credentials from our provider are valid - builder.credentialsProvider(provider) - .build - .getCallerIdentity(GetCallerIdentityRequest.builder.build) - () - } - - protected def validateCredential(provider: AwsCredentialsProvider, region: Option[String]) = { + private[auth] var credentialValidation: (AwsCredentialsProvider, Option[String]) => Unit = + (provider: AwsCredentialsProvider, region: Option[String]) => { + val builder = StsClient.builder + + // If the region argument exists in config, set it in the builder. + // Otherwise it is left unset and the AwsCredentialsProvider will be responsible for sourcing a region + region.map(Region.of).foreach(builder.region) + + // make an essentially no-op call just to assure ourselves the credentials from our provider are valid + builder + .credentialsProvider(provider) + .build + .getCallerIdentity(GetCallerIdentityRequest.builder.build) + () + } + + protected def validateCredential(provider: AwsCredentialsProvider, region: Option[String]) = Try(credentialValidation(provider, region)) match { - case Failure(ex) => throw new RuntimeException(s"Credentials produced by the AWS provider ${name} are invalid: ${ex.getMessage}", ex) + case Failure(ex) => + throw new RuntimeException(s"Credentials produced by the AWS provider ${name} are invalid: ${ex.getMessage}", + ex + ) case Success(_) => provider } - } } /** @@ -114,11 +116,8 @@ object CustomKeyMode * @param secretKey static AWS secret key * @param region an optional AWS region */ -final case class CustomKeyMode(override val name: String, - accessKey: String, - secretKey: String, - region: Option[String] - ) extends AwsAuthMode { +final case class CustomKeyMode(override val name: String, accessKey: String, secretKey: String, region: Option[String]) + extends AwsAuthMode { private lazy val _provider: AwsCredentialsProvider = { // make a provider locked to the given access and secret val p = StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey, secretKey)) @@ -159,17 +158,17 @@ final case class DefaultMode(override val name: String, region: Option[String]) * @param region an optional AWS region */ final case class AssumeRoleMode(override val name: String, - baseAuthName: String, - roleArn: String, - externalId: String, - region: Option[String] - ) extends AwsAuthMode { + baseAuthName: String, + roleArn: String, + externalId: String, + region: Option[String] +) extends AwsAuthMode { private lazy val _provider: AwsCredentialsProvider = { // we need to perform operations on STS using the credentials provided from the baseAuthName val stsBuilder = StsClient.builder region.foreach(str => stsBuilder.region(Region.of(str))) - baseAuthObj match{ + baseAuthObj match { case Some(auth) => stsBuilder.credentialsProvider(auth.provider()) case _ => throw new RuntimeException(s"Base auth configuration required for assume role") } @@ -179,7 +178,7 @@ final case class AssumeRoleMode(override val name: String, .roleArn(roleArn) .durationSeconds(3600) .roleSessionName("cromwell") - if (! externalId.isEmpty) assumeRoleBuilder.externalId(externalId) + if (!externalId.isEmpty) assumeRoleBuilder.externalId(externalId) // this provider is one that will handle refreshing the assume-role creds when needed val p = StsAssumeRoleCredentialsProvider.builder @@ -195,23 +194,21 @@ final case class AssumeRoleMode(override val name: String, // start a background thread to perform the refresh override def provider(): AwsCredentialsProvider = _provider - private var baseAuthObj : Option[AwsAuthMode] = None + private var baseAuthObj: Option[AwsAuthMode] = None - def assign(baseAuth: AwsAuthMode) : Unit = { + def assign(baseAuth: AwsAuthMode): Unit = baseAuthObj match { case None => baseAuthObj = Some(baseAuth) case _ => throw new RuntimeException(s"Base auth object has already been assigned") } - } // We want to allow our tests access to the value // of the baseAuthObj - def baseAuthentication() : AwsAuthMode = { + def baseAuthentication(): AwsAuthMode = baseAuthObj match { case Some(o) => o case _ => throw new RuntimeException(s"Base auth object has not been set") } - } } class OptionLookupException(val key: String, cause: Throwable) extends RuntimeException(key, cause) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala index 8ab07f37064..8caf8aa3fb1 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/aws/s3/S3Storage.scala @@ -60,13 +60,13 @@ object S3Storage { builder.build } - def s3Client(provider: AwsCredentialsProvider, region: Option[Region]): S3Client = { + def s3Client(provider: AwsCredentialsProvider, region: Option[Region]): S3Client = s3Client(s3Configuration(), provider, region) - } def s3Configuration(accelerateModeEnabled: Boolean = false, dualstackEnabled: Boolean = false, - pathStyleAccessEnabled: Boolean = false): S3Configuration = { + pathStyleAccessEnabled: Boolean = false + ): S3Configuration = { @nowarn("msg=method dualstackEnabled in trait Builder is deprecated") val builder = S3Configuration.builder diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala new file mode 100644 index 00000000000..d3d66e1bafc --- /dev/null +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureCredentials.scala @@ -0,0 +1,53 @@ +package cromwell.cloudsupport.azure + +import cats.implicits.catsSyntaxValidatedId +import com.azure.core.credential.TokenRequestContext +import com.azure.core.management.AzureEnvironment +import com.azure.core.management.profile.AzureProfile +import com.azure.identity.DefaultAzureCredentialBuilder +import common.validation.ErrorOr.ErrorOr + +import scala.concurrent.duration._ +import scala.jdk.DurationConverters._ +import scala.util.{Failure, Success, Try} + +/** + * Strategy for obtaining an access token in an environment with available Azure identity. + * If you need to disambiguate among multiple active user-assigned managed identities, pass + * in the client id of the identity that should be used. + */ +case object AzureCredentials { + + final val tokenAcquisitionTimeout = 5.seconds + + val azureProfile = new AzureProfile(AzureEnvironment.AZURE) + val tokenScope = "https://management.azure.com/.default" + + private def tokenRequestContext: TokenRequestContext = { + val trc = new TokenRequestContext() + trc.addScopes(tokenScope) + trc + } + + private def defaultCredentialBuilder: DefaultAzureCredentialBuilder = + new DefaultAzureCredentialBuilder() + .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint) + + def getAccessToken(identityClientId: Option[String] = None): ErrorOr[String] = { + val credentials = identityClientId + .foldLeft(defaultCredentialBuilder) { (builder, clientId) => + builder.managedIdentityClientId(clientId) + } + .build() + + Try( + credentials + .getToken(tokenRequestContext) + .block(tokenAcquisitionTimeout.toJava) + ) match { + case Success(null) => "null token value attempting to obtain access token".invalidNel + case Success(token) => token.getToken.validNel + case Failure(error) => s"Failed to refresh access token: ${error.getMessage}".invalidNel + } + } +} diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala new file mode 100644 index 00000000000..dd379ed3564 --- /dev/null +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/azure/AzureUtils.scala @@ -0,0 +1,80 @@ +package cromwell.cloudsupport.azure + +import com.azure.core.management.AzureEnvironment +import com.azure.core.management.profile.AzureProfile +import com.azure.identity.DefaultAzureCredentialBuilder +import com.azure.resourcemanager.AzureResourceManager +import com.azure.resourcemanager.storage.models.StorageAccountKey +import com.azure.storage.blob.{BlobContainerClient, BlobContainerClientBuilder} +import com.azure.storage.common.StorageSharedKeyCredential +import com.google.common.net.UrlEscapers + +import java.net.URI +import scala.jdk.CollectionConverters.IterableHasAsScala +import scala.util.{Failure, Success, Try} + +object AzureUtils { + + /** + * Generates a BlobContainerClient that can interact with the specified container. Authenticates using the local azure client running on the same machine. + * @param blobContainer Name of the blob container. Looks something like "my-blob-container". + * @param azureEndpoint Azure endpoint of the container. Looks something like https://somedomain.blob.core.windows.net. + * @param subscription Azure subscription. A globally unique identifier. If not provided, a default subscription will be used. + * @return A blob container client capable of interacting with the specified container. + */ + def buildContainerClientFromLocalEnvironment(blobContainer: String, + azureEndpoint: String, + subscription: Option[String] + ): Try[BlobContainerClient] = { + def parseURI(string: String): Try[URI] = Try(URI.create(UrlEscapers.urlFragmentEscaper().escape(string))) + def parseStorageAccount(uri: URI): Try[String] = uri.getHost + .split("\\.") + .find(_.nonEmpty) + .map(Success(_)) + .getOrElse(Failure(new Exception("Could not parse storage account"))) + + val azureProfile = new AzureProfile(AzureEnvironment.AZURE) + + def azureCredentialBuilder = new DefaultAzureCredentialBuilder() + .authorityHost(azureProfile.getEnvironment.getActiveDirectoryEndpoint) + .build + + def authenticateWithSubscription(sub: String) = + AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withSubscription(sub) + + def authenticateWithDefaultSubscription = + AzureResourceManager.authenticate(azureCredentialBuilder, azureProfile).withDefaultSubscription() + + def azure = subscription.map(authenticateWithSubscription(_)).getOrElse(authenticateWithDefaultSubscription) + + def findAzureStorageAccount(storageAccountName: String) = azure.storageAccounts.list.asScala + .find(_.name.equals(storageAccountName)) + .map(Success(_)) + .getOrElse(Failure(new Exception("Azure Storage Account not found."))) + + def buildBlobContainerClient(credential: StorageSharedKeyCredential, + endpointURL: String, + blobContainerName: String + ): BlobContainerClient = + new BlobContainerClientBuilder() + .credential(credential) + .endpoint(endpointURL) + .containerName(blobContainerName) + .buildClient() + + def generateBlobContainerClient: Try[BlobContainerClient] = for { + uri <- parseURI(azureEndpoint) + configuredAccount <- parseStorageAccount(uri) + azureAccount <- findAzureStorageAccount(configuredAccount) + keys = azureAccount.getKeys.asScala + key <- keys.headOption.fold[Try[StorageAccountKey]](Failure(new Exception("Storage account has no keys")))( + Success(_) + ) + first = key.value + sskc = new StorageSharedKeyCredential(configuredAccount, first) + bcc = buildBlobContainerClient(sskc, azureEndpoint, blobContainer) + } yield bcc + + generateBlobContainerClient + } +} diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala index 2b4a183c121..999a9c2867a 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/GoogleConfiguration.scala @@ -18,14 +18,13 @@ import org.slf4j.LoggerFactory final case class GoogleConfiguration private (applicationName: String, authsByName: Map[String, GoogleAuthMode]) { - def auth(name: String): ErrorOr[GoogleAuthMode] = { + def auth(name: String): ErrorOr[GoogleAuthMode] = authsByName.get(name) match { case None => val knownAuthNames = authsByName.keys.mkString(", ") s"`google` configuration stanza does not contain an auth named '$name'. Known auth names: $knownAuthNames".invalidNel case Some(a) => a.validNel } - } } object GoogleConfiguration { @@ -37,7 +36,8 @@ object GoogleConfiguration { def withCustomTimeouts(httpRequestInitializer: HttpRequestInitializer, connectionTimeout: FiniteDuration = DefaultConnectionTimeout, - readTimeout: FiniteDuration = DefaultReadTimeout): HttpRequestInitializer = { + readTimeout: FiniteDuration = DefaultReadTimeout + ): HttpRequestInitializer = new HttpRequestInitializer() { @throws[IOException] override def initialize(httpRequest: HttpRequest): Unit = { @@ -47,7 +47,6 @@ object GoogleConfiguration { () } } - } private val log = LoggerFactory.getLogger("GoogleConfiguration") @@ -59,20 +58,27 @@ object GoogleConfiguration { val googleConfig = config.getConfig("google") - val appName = validate { googleConfig.as[String]("application-name") } + val appName = validate(googleConfig.as[String]("application-name")) def buildAuth(authConfig: Config): ErrorOr[GoogleAuthMode] = { def serviceAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { (authConfig.getAs[String]("pem-file"), authConfig.getAs[String]("json-file")) match { - case (Some(pem), None) => ServiceAccountMode(name, PemFileFormat(authConfig.as[String]("service-account-id"), pem)) + case (Some(pem), None) => + ServiceAccountMode(name, PemFileFormat(authConfig.as[String]("service-account-id"), pem)) case (None, Some(json)) => ServiceAccountMode(name, JsonFileFormat(json)) - case (None, None) => throw new ConfigException.Generic(s"""No credential configuration was found for service account "$name". See reference.conf under the google.auth, service-account section for supported credential formats.""") - case (Some(_), Some(_)) => throw new ConfigException.Generic(s"""Both a pem file and a json file were supplied for service account "$name" in the configuration file. Only one credential file can be supplied for the same service account. Please choose between the two.""") + case (None, None) => + throw new ConfigException.Generic( + s"""No credential configuration was found for service account "$name". See reference.conf under the google.auth, service-account section for supported credential formats.""" + ) + case (Some(_), Some(_)) => + throw new ConfigException.Generic( + s"""Both a pem file and a json file were supplied for service account "$name" in the configuration file. Only one credential file can be supplied for the same service account. Please choose between the two.""" + ) } } - def userAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { + def userAccountAuth(authConfig: Config, name: String): ErrorOr[GoogleAuthMode] = validate { UserMode(name, authConfig.as[String]("secrets-file")) } diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala index e850b53807a..52118303262 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthMode.scala @@ -34,9 +34,8 @@ object GoogleAuthMode { type CredentialsValidation = Credentials => Unit private[auth] val NoCredentialsValidation = mouse.ignore _ - private def noOptionLookup(string: String): Nothing = { + private def noOptionLookup(string: String): Nothing = throw new UnsupportedOperationException(s"cannot lookup $string") - } lazy val jsonFactory: GsonFactory = GsonFactory.getDefaultInstance lazy val httpTransport: HttpTransport = GoogleNetHttpTransport.newTrustedTransport @@ -46,33 +45,29 @@ object GoogleAuthMode { val DockerCredentialsEncryptionKeyNameKey = "docker_credentials_key_name" val DockerCredentialsTokenKey = "docker_credentials_token" - def checkReadable(file: File): Unit = { + def checkReadable(file: File): Unit = if (!file.isReadable) throw new FileNotFoundException(s"File $file does not exist or is not readable") - } - def isFatal(ex: Throwable): Boolean = { + def isFatal(ex: Throwable): Boolean = ex match { case http: HttpResponseException => // Using HttpURLConnection fields as com.google.api.client.http.HttpStatusCodes doesn't have Bad Request (400) http.getStatusCode == HTTP_UNAUTHORIZED || - http.getStatusCode == HTTP_FORBIDDEN || - http.getStatusCode == HTTP_BAD_REQUEST + http.getStatusCode == HTTP_FORBIDDEN || + http.getStatusCode == HTTP_BAD_REQUEST case _: OptionLookupException => true case _ => false } - } - def extract(options: OptionLookup, key: String): String = { + def extract(options: OptionLookup, key: String): String = Try(options(key)) match { case Success(result) => result case Failure(throwable) => throw new OptionLookupException(key, throwable) } - } /** Used for both checking that the credential is valid and creating a fresh credential. */ - private def refreshCredentials(credentials: Credentials): Unit = { + private def refreshCredentials(credentials: Credentials): Unit = credentials.refresh() - } } sealed trait GoogleAuthMode extends LazyLogging { @@ -87,25 +82,22 @@ sealed trait GoogleAuthMode extends LazyLogging { * Alias for credentials(GoogleAuthMode.NoOptionLookup, scopes). * Only valid for credentials that are NOT externally provided, such as ApplicationDefault. */ - def credentials(scopes: Iterable[String]): OAuth2Credentials = { + def credentials(scopes: Iterable[String]): OAuth2Credentials = credentials(GoogleAuthMode.NoOptionLookup, scopes) - } /** * Alias for credentials(GoogleAuthMode.NoOptionLookup, Nil). * Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault. */ - private[auth] def credentials(): OAuth2Credentials = { + private[auth] def credentials(): OAuth2Credentials = credentials(GoogleAuthMode.NoOptionLookup, Nil) - } /** * Alias for credentials(options, Nil). * Only valid for credentials that are NOT externally provided and do not need scopes, such as ApplicationDefault. */ - private[auth] def credentials(options: OptionLookup): OAuth2Credentials = { + private[auth] def credentials(options: OptionLookup): OAuth2Credentials = credentials(options, Nil) - } /** * Enables swapping out credential validation for various testing purposes ONLY. @@ -116,7 +108,8 @@ sealed trait GoogleAuthMode extends LazyLogging { private[auth] var credentialsValidation: CredentialsValidation = refreshCredentials protected def validateCredentials[A <: GoogleCredentials](credential: A, - scopes: Iterable[String]): GoogleCredentials = { + scopes: Iterable[String] + ): GoogleCredentials = { val scopedCredentials = credential.createScoped(scopes.asJavaCollection) Try(credentialsValidation(scopedCredentials)) match { case Failure(ex) => throw new RuntimeException(s"Google credentials are invalid: ${ex.getMessage}", ex) @@ -126,9 +119,8 @@ sealed trait GoogleAuthMode extends LazyLogging { } case class MockAuthMode(override val name: String) extends GoogleAuthMode { - override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): NoCredentials = { + override def credentials(unusedOptions: OptionLookup, unusedScopes: Iterable[String]): NoCredentials = NoCredentials.getInstance - } } object ServiceAccountMode { @@ -143,35 +135,29 @@ object ServiceAccountMode { } -final case class ServiceAccountMode(override val name: String, - fileFormat: CredentialFileFormat) - extends GoogleAuthMode { +final case class ServiceAccountMode(override val name: String, fileFormat: CredentialFileFormat) + extends GoogleAuthMode { private val credentialsFile = File(fileFormat.file) checkReadable(credentialsFile) - private lazy val serviceAccountCredentials: ServiceAccountCredentials = { + private lazy val serviceAccountCredentials: ServiceAccountCredentials = fileFormat match { case PemFileFormat(accountId, _) => logger.warn("The PEM file format will be deprecated in the upcoming Cromwell version. Please use JSON instead.") ServiceAccountCredentials.fromPkcs8(accountId, accountId, credentialsFile.contentAsString, null, null) case _: JsonFileFormat => ServiceAccountCredentials.fromStream(credentialsFile.newInputStream) } - } - override def credentials(unusedOptions: OptionLookup, - scopes: Iterable[String]): GoogleCredentials = { + override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = validateCredentials(serviceAccountCredentials, scopes) - } } final case class UserServiceAccountMode(override val name: String) extends GoogleAuthMode { - private def extractServiceAccount(options: OptionLookup): String = { + private def extractServiceAccount(options: OptionLookup): String = extract(options, UserServiceAccountKey) - } - private def credentialStream(options: OptionLookup): InputStream = { + private def credentialStream(options: OptionLookup): InputStream = new ByteArrayInputStream(extractServiceAccount(options).getBytes(StandardCharsets.UTF_8)) - } override def credentials(options: OptionLookup, scopes: Iterable[String]): GoogleCredentials = { val newCredentials = ServiceAccountCredentials.fromStream(credentialStream(options)) @@ -179,7 +165,6 @@ final case class UserServiceAccountMode(override val name: String) extends Googl } } - final case class UserMode(override val name: String, secretsPath: String) extends GoogleAuthMode { private lazy val secretsStream = { @@ -190,9 +175,8 @@ final case class UserMode(override val name: String, secretsPath: String) extend private lazy val userCredentials: UserCredentials = UserCredentials.fromStream(secretsStream) - override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = { + override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = validateCredentials(userCredentials, scopes) - } } object ApplicationDefaultMode { @@ -200,10 +184,8 @@ object ApplicationDefaultMode { } final case class ApplicationDefaultMode(name: String) extends GoogleAuthMode { - override def credentials(unusedOptions: OptionLookup, - scopes: Iterable[String]): GoogleCredentials = { + override def credentials(unusedOptions: OptionLookup, scopes: Iterable[String]): GoogleCredentials = validateCredentials(applicationDefaultCredentials, scopes) - } } sealed trait ClientSecrets { diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala index 29ed842377d..6ae6e88542c 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/gcs/GcsStorage.scala @@ -18,9 +18,11 @@ object GcsStorage { val HttpTransport = GoogleNetHttpTransport.newTrustedTransport val DefaultCloudStorageConfiguration = { - val UploadBufferBytes = ConfigFactory.load().as[Option[Int]]("google.upload-buffer-bytes").getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) + val UploadBufferBytes = + ConfigFactory.load().as[Option[Int]]("google.upload-buffer-bytes").getOrElse(MediaHttpUploader.MINIMUM_CHUNK_SIZE) - CloudStorageConfiguration.builder() + CloudStorageConfiguration + .builder() .blockSize(UploadBufferBytes) .permitEmptyPathComponents(true) .stripPrefixSlash(true) @@ -28,23 +30,24 @@ object GcsStorage { .build() } - def gcsStorage(applicationName: String, - storageOptions: StorageOptions): Storage = { - new Storage.Builder(HttpTransport, + def gcsStorage(applicationName: String, storageOptions: StorageOptions): Storage = + new Storage.Builder( + HttpTransport, JsonFactory, - GoogleConfiguration.withCustomTimeouts(TransportOptions.getHttpRequestInitializer(storageOptions))) + GoogleConfiguration.withCustomTimeouts(TransportOptions.getHttpRequestInitializer(storageOptions)) + ) .setApplicationName(applicationName) .build() - } - def gcsStorage(applicationName: String, credentials: Credentials, retrySettings: RetrySettings): Storage = { + def gcsStorage(applicationName: String, credentials: Credentials, retrySettings: RetrySettings): Storage = gcsStorage(applicationName, gcsStorageOptions(credentials, retrySettings)) - } def gcsStorageOptions(credentials: Credentials, retrySettings: RetrySettings, - project: Option[String] = None): StorageOptions = { - val storageOptionsBuilder = StorageOptions.newBuilder() + project: Option[String] = None + ): StorageOptions = { + val storageOptionsBuilder = StorageOptions + .newBuilder() .setTransportOptions(TransportOptions) .setCredentials(credentials) diff --git a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala index 13faef01e0c..f328756cf67 100644 --- a/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala +++ b/cloudSupport/src/main/scala/cromwell/cloudsupport/gcp/http/GoogleHttpTransportOptions.scala @@ -4,7 +4,8 @@ import com.google.cloud.http.HttpTransportOptions import scala.concurrent.duration._ object GoogleHttpTransportOptions { - val TransportOptions = HttpTransportOptions.newBuilder() + val TransportOptions = HttpTransportOptions + .newBuilder() .setReadTimeout(3.minutes.toMillis.toInt) .build() } diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala index 6ead310c925..27c66305466 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/AwsConfigurationSpec.scala @@ -39,7 +39,6 @@ import cromwell.cloudsupport.aws.auth.{AssumeRoleMode, CustomKeyMode, DefaultMod import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class AwsConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "AwsConfiguration" @@ -47,38 +46,38 @@ class AwsConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat it should "parse all manner of well-formed auths" in { val righteousAwsConfig = s""" - |aws { - | application-name = "cromwell" - | - | auths = [ - | { - | name = "default" - | scheme = "default" - | }, - | { - | name = "custom-keys" - | scheme = "custom_keys" - | access-key = "access_key_id" - | secret-key = "secret_key" - | }, - | { - | name = "assume-role-based-on-another-with-external" - | scheme = "assume_role" - | base-auth = "default" - | role-arn = "my-role-arn" - | external-id = "my-external-id" - | }, - | { - | name = "assume-role-based-on-another" - | scheme = "assume_role" - | base-auth = "default" - | role-arn = "my-role-arn" - | } - | ] - | - | region = "region" - |} - | + |aws { + | application-name = "cromwell" + | + | auths = [ + | { + | name = "default" + | scheme = "default" + | }, + | { + | name = "custom-keys" + | scheme = "custom_keys" + | access-key = "access_key_id" + | secret-key = "secret_key" + | }, + | { + | name = "assume-role-based-on-another-with-external" + | scheme = "assume_role" + | base-auth = "default" + | role-arn = "my-role-arn" + | external-id = "my-external-id" + | }, + | { + | name = "assume-role-based-on-another" + | scheme = "assume_role" + | base-auth = "default" + | role-arn = "my-role-arn" + | } + | ] + | + | region = "region" + |} + | """.stripMargin val conf = AwsConfiguration(ConfigFactory.parseString(righteousAwsConfig)) @@ -146,8 +145,8 @@ class AwsConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat val conf = AwsConfiguration(ConfigFactory.parseString(config)) conf.auth("name-botched") should be( - "`aws` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default" - .invalidNel) + "`aws` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default".invalidNel + ) } it should "not parse a configuration stanza without applicationName" in { diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala index 5311714e89c..e50b24d928f 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/aws/s3/S3StorageSpec.scala @@ -36,7 +36,6 @@ import org.scalatest.matchers.should.Matchers import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider import software.amazon.awssdk.regions.Region - class S3StorageSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "S3Storage" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala index 95a2380034a..7ed9162a6d1 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/GoogleConfigurationSpec.scala @@ -14,7 +14,6 @@ import cromwell.cloudsupport.gcp.auth._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class GoogleConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "GoogleConfiguration" @@ -25,39 +24,39 @@ class GoogleConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with val righteousGoogleConfig = s""" - |google { - | application-name = "cromwell" - | - | auths = [ - | { - | name = "name-default" - | scheme = "application_default" - | }, - | { - | name = "name-user" - | scheme = "user_account" - | user = "me" - | secrets-file = "${pemMockFile.pathAsString}" - | data-store-dir = "/where/the/data/at" - | }, - | { - | name = "name-pem-service" - | scheme = "service_account" - | service-account-id = "my-google-account" - | pem-file = "${pemMockFile.pathAsString}" - | }, - | { - | name = "name-json-service" - | scheme = "service_account" - | json-file = "${jsonMockFile.pathAsString}" - | }, - | { - | name = "name-user-service-account" - | scheme = "user_service_account" - | } - | ] - |} - | + |google { + | application-name = "cromwell" + | + | auths = [ + | { + | name = "name-default" + | scheme = "application_default" + | }, + | { + | name = "name-user" + | scheme = "user_account" + | user = "me" + | secrets-file = "${pemMockFile.pathAsString}" + | data-store-dir = "/where/the/data/at" + | }, + | { + | name = "name-pem-service" + | scheme = "service_account" + | service-account-id = "my-google-account" + | pem-file = "${pemMockFile.pathAsString}" + | }, + | { + | name = "name-json-service" + | scheme = "service_account" + | json-file = "${jsonMockFile.pathAsString}" + | }, + | { + | name = "name-user-service-account" + | scheme = "user_service_account" + | } + | ] + |} + | """.stripMargin val gconf = GoogleConfiguration(ConfigFactory.parseString(righteousGoogleConfig)) @@ -125,16 +124,16 @@ class GoogleConfigurationSpec extends AnyFlatSpec with CromwellTimeoutSpec with val googleConfiguration = GoogleConfiguration(ConfigFactory.parseString(config)) googleConfiguration.auth("name-botched") should be( - "`google` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default" - .invalidNel) + "`google` configuration stanza does not contain an auth named 'name-botched'. Known auth names: name-default".invalidNel + ) } it should "create an initializer with custom timeouts" in { val transport = new MockHttpTransport() - val initializer = GoogleConfiguration.withCustomTimeouts(request => { + val initializer = GoogleConfiguration.withCustomTimeouts { request => request.getHeaders.set("custom_init", "ok") () - }) + } val factory = transport.createRequestFactory(initializer) val request = factory.buildGetRequest(new GenericUrl(new URL("http://example.com"))) request.getConnectTimeout should be(180000) diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala index bb661ce9742..dde1f82a3a5 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ApplicationDefaultModeSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class ApplicationDefaultModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "ApplicationDefaultMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala index 368ce5d2472..bf77f65850d 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/GoogleAuthModeSpec.scala @@ -10,14 +10,12 @@ import org.scalatest.prop.TableDrivenPropertyChecks import scala.util.{Failure, Try} - class GoogleAuthModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with TableDrivenPropertyChecks { behavior of "GoogleAuthMode" - private def mockHttpResponseException(statusCode: Int): HttpResponseException = { + private def mockHttpResponseException(statusCode: Int): HttpResponseException = new HttpResponseException.Builder(statusCode, "mock message", new HttpHeaders).build() - } private val testedExceptions = Table( ("description", "exception", "isFatal"), @@ -55,14 +53,13 @@ object GoogleAuthModeSpec extends ServiceAccountTestSupport { () } - lazy val userCredentialsContents: String = { + lazy val userCredentialsContents: String = toJson( "type" -> "authorized_user", "client_id" -> "the_id", "client_secret" -> "the_secret", "refresh_token" -> "the_token" ) - } lazy val refreshTokenOptions: OptionLookup = Map("refresh_token" -> "the_refresh_token") lazy val userServiceAccountOptions: OptionLookup = Map("user_service_account_json" -> serviceAccountJsonContents) diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala index 591da3354a7..b0c89e368c5 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/MockAuthModeSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class MockAuthModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "MockAuthMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala index 94831b1410b..b10a823a6ca 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountModeSpec.scala @@ -30,7 +30,7 @@ class ServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with M .write(serviceAccountPemContents) val serviceAccountMode = ServiceAccountMode( "service-account", - ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString), + ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString) ) val exception = intercept[RuntimeException](serviceAccountMode.credentials()) exception.getMessage should startWith("Google credentials are invalid: ") @@ -53,7 +53,7 @@ class ServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with M val exception = intercept[FileNotFoundException] { ServiceAccountMode( "service-account", - ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString), + ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString) ) } exception.getMessage should fullyMatch regex "File .*/service-account..*.pem does not exist or is not readable" @@ -79,7 +79,7 @@ class ServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with M .write(serviceAccountPemContents) val serviceAccountMode = ServiceAccountMode( "service-account", - ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString), + ServiceAccountMode.PemFileFormat("the_account_id", pemMockFile.pathAsString) ) serviceAccountMode.credentialsValidation = GoogleAuthMode.NoCredentialsValidation val credentials = serviceAccountMode.credentials() diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala index 1624674b875..a87df58e21d 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/ServiceAccountTestSupport.scala @@ -23,7 +23,7 @@ trait ServiceAccountTestSupport { // Hide me from git secrets false positives private val theStringThatShallNotBeNamed = List("private", "key").mkString("_") - def serviceAccountJsonContents: String = { + def serviceAccountJsonContents: String = toJson( "type" -> "service_account", "client_id" -> "the_account_id", @@ -31,7 +31,6 @@ trait ServiceAccountTestSupport { theStringThatShallNotBeNamed -> serviceAccountPemContents, s"${theStringThatShallNotBeNamed}_id" -> "the_key_id" ) - } def toJson(contents: (String, String)*): String = { // Generator doesn't matter as long as it generates JSON. Using `jsonFactory` to get an extra line hit of coverage. @@ -40,10 +39,9 @@ trait ServiceAccountTestSupport { val generator = factory.createJsonGenerator(writer) generator.enablePrettyPrint() generator.writeStartObject() - contents foreach { - case (key, value) => - generator.writeFieldName(key) - generator.writeString(value) + contents foreach { case (key, value) => + generator.writeFieldName(key) + generator.writeString(value) } generator.writeEndObject() generator.close() diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala index cfc22b0ca9b..95a64ee1e6c 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserModeSpec.scala @@ -7,7 +7,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class UserModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "UserMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala index 70a1f470d32..092072f7511 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/auth/UserServiceAccountModeSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class UserServiceAccountModeSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "UserServiceAccountMode" diff --git a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala index 558c9b74c4a..1f673acec67 100644 --- a/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala +++ b/cloudSupport/src/test/scala/cromwell/cloudsupport/gcp/gcs/GcsStorageSpec.scala @@ -6,7 +6,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class GcsStorageSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "GcsStorage" @@ -19,7 +18,8 @@ class GcsStorageSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers } it should "build gcs storage" in { - val configuration = GcsStorage.gcsStorage("gcs-storage-spec", NoCredentials.getInstance(), RetrySettings.newBuilder().build()) + val configuration = + GcsStorage.gcsStorage("gcs-storage-spec", NoCredentials.getInstance(), RetrySettings.newBuilder().build()) configuration.getApplicationName should be("gcs-storage-spec") } diff --git a/codegen_java/build.sbt b/codegen_java/build.sbt index 2ba35a693c3..0ad461a66b8 100644 --- a/codegen_java/build.sbt +++ b/codegen_java/build.sbt @@ -6,7 +6,7 @@ lazy val root = (project in file(".")). Seq(organization := "org.broadinstitute.cromwell", name := "cromwell-client", version := createVersion("0.1"), - scalaVersion := "2.13.8", + scalaVersion := "2.13.9", scalacOptions ++= Seq("-feature"), compile / javacOptions ++= Seq("-Xlint:deprecation"), Compile / packageDoc / publishArtifact := false, diff --git a/codegen_java/project/Artifactory.scala b/codegen_java/project/Artifactory.scala index a13c9cc8c21..6d385ffbbc7 100644 --- a/codegen_java/project/Artifactory.scala +++ b/codegen_java/project/Artifactory.scala @@ -1,4 +1,4 @@ object Artifactory { val artifactoryHost = "broadinstitute.jfrog.io" val artifactory = s"https://$artifactoryHost/broadinstitute/" -} \ No newline at end of file +} diff --git a/codegen_java/project/Publishing.scala b/codegen_java/project/Publishing.scala index 6c3de9881fc..880cf40f1ff 100644 --- a/codegen_java/project/Publishing.scala +++ b/codegen_java/project/Publishing.scala @@ -2,10 +2,10 @@ import sbt.Keys._ import sbt._ import Artifactory._ - object Publishing { +object Publishing { private val buildTimestamp = System.currentTimeMillis() / 1000 - private def artifactoryResolver(isSnapshot: Boolean): Resolver = { + private def artifactoryResolver(isSnapshot: Boolean): Resolver = { val repoType = if (isSnapshot) "snapshot" else "release" val repoUrl = s"${artifactory}libs-$repoType-local;build.timestamp=$buildTimestamp" @@ -13,15 +13,15 @@ import Artifactory._ repoName at repoUrl } - private val artifactoryCredentials: Credentials = { + private val artifactoryCredentials: Credentials = { val username = sys.env.getOrElse("ARTIFACTORY_USERNAME", "") val password = sys.env.getOrElse("ARTIFACTORY_PASSWORD", "") Credentials("Artifactory Realm", artifactoryHost, username, password) } - val publishSettings: Seq[Setting[_]] = - //we only publish to libs-release-local because of a bug in sbt that makes snapshots take - //priority over the local package cache. see here: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 + val publishSettings: Seq[Setting[_]] = + // we only publish to libs-release-local because of a bug in sbt that makes snapshots take + // priority over the local package cache. see here: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 Seq( publishTo := Option(artifactoryResolver(false)), Compile / publishArtifact := true, @@ -29,9 +29,9 @@ import Artifactory._ credentials += artifactoryCredentials ) - val noPublishSettings: Seq[Setting[_]] = + val noPublishSettings: Seq[Setting[_]] = Seq( publish := {}, publishLocal := {} ) -} \ No newline at end of file +} diff --git a/codegen_java/project/Version.scala b/codegen_java/project/Version.scala index 875a80e2e10..be7c57bc527 100644 --- a/codegen_java/project/Version.scala +++ b/codegen_java/project/Version.scala @@ -1,20 +1,20 @@ import scala.sys.process._ - object Version { +object Version { - def createVersion(baseVersion: String) = { - def getLastCommitFromGit = { s"""git rev-parse --short HEAD""" !! } + def createVersion(baseVersion: String) = { + def getLastCommitFromGit = s"""git rev-parse --short HEAD""" !! - // either specify git hash as an env var or derive it + // either specify git hash as an env var or derive it // if building from the broadinstitute/scala-baseimage docker image use env var // (scala-baseimage doesn't have git in it) - val lastCommit = sys.env.getOrElse("GIT_HASH", getLastCommitFromGit ).trim() + val lastCommit = sys.env.getOrElse("GIT_HASH", getLastCommitFromGit).trim() val version = baseVersion + "-" + lastCommit - // The project isSnapshot string passed in via command line settings, if desired. + // The project isSnapshot string passed in via command line settings, if desired. val isSnapshot = sys.props.getOrElse("project.isSnapshot", "true").toBoolean - // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 + // For now, obfuscate SNAPSHOTs from sbt's developers: https://github.com/sbt/sbt/issues/2687#issuecomment-236586241 if (isSnapshot) s"$version-SNAP" else version } -} \ No newline at end of file +} diff --git a/common/src/main/scala/common/collections/EnhancedCollections.scala b/common/src/main/scala/common/collections/EnhancedCollections.scala index beede1f9510..e1eb983029c 100644 --- a/common/src/main/scala/common/collections/EnhancedCollections.scala +++ b/common/src/main/scala/common/collections/EnhancedCollections.scala @@ -15,7 +15,10 @@ object EnhancedCollections { * After trying and failing to do this myself, I got this to work by copying the answer from here: * https://stackoverflow.com/questions/29886246/scala-filter-by-type */ - implicit class EnhancedIterableOps[T2, Repr[x] <: IterableOps[x, Repr, Repr[x]]](val iterableOps: IterableOps[T2, Repr, Repr[T2]]) extends AnyVal { + implicit class EnhancedIterableOps[T2, Repr[x] <: IterableOps[x, Repr, Repr[x]]]( + val iterableOps: IterableOps[T2, Repr, Repr[T2]] + ) extends AnyVal { + /** * Lets you filter a collection by type. * @@ -58,27 +61,29 @@ object EnhancedCollections { def takeWhileWeighted[W](maxWeight: W, weightFunction: A => W, maxHeadLength: Option[Int], - strict: Boolean = false) - (implicit n: Numeric[W], c: Ordering[W]): DeQueued[A] = { + strict: Boolean = false + )(implicit n: Numeric[W], c: Ordering[W]): DeQueued[A] = { import n._ @tailrec - def takeWhileWeightedRec(tail: Queue[A], head: Vector[A], weight: W): (Vector[A], Queue[A]) = { + def takeWhileWeightedRec(tail: Queue[A], head: Vector[A], weight: W): (Vector[A], Queue[A]) = // Stay under maxHeadLength if it's specified if (maxHeadLength.exists(head.size >= _)) head -> tail - else tail.dequeueOption - .map({ - // Compute the dequeued element's weight - case (element, dequeued) => (element, weightFunction(element), dequeued) - }) match { - // If the element's weight is > maxWeight and strict is true, drop the element - case Some((_, elementWeight, dequeued)) if c.gteq(elementWeight, maxWeight) && strict => takeWhileWeightedRec(dequeued, head, weight) - // If we're under the max weight, add the element to the head and recurse - case Some((element, elementWeight, dequeued)) if c.lteq(elementWeight + weight, maxWeight) => takeWhileWeightedRec(dequeued, head :+ element, weight + elementWeight) - // Otherwise stop here (make sure to return the original queue so we don't lose the last dequeued element) - case _ => head -> tail - } - } + else + tail.dequeueOption + .map { + // Compute the dequeued element's weight + case (element, dequeued) => (element, weightFunction(element), dequeued) + } match { + // If the element's weight is > maxWeight and strict is true, drop the element + case Some((_, elementWeight, dequeued)) if c.gteq(elementWeight, maxWeight) && strict => + takeWhileWeightedRec(dequeued, head, weight) + // If we're under the max weight, add the element to the head and recurse + case Some((element, elementWeight, dequeued)) if c.lteq(elementWeight + weight, maxWeight) => + takeWhileWeightedRec(dequeued, head :+ element, weight + elementWeight) + // Otherwise stop here (make sure to return the original queue so we don't lose the last dequeued element) + case _ => head -> tail + } if (queue.isEmpty || maxHeadLength.contains(0)) DeQueued(Vector.empty, queue) // If strict is enabled, we should never return a head with a weight > maxWeight. So start from the original queue and drop elements over maxWeight if necessary @@ -88,13 +93,17 @@ object EnhancedCollections { } // Otherwise to ensure we don't deadlock, start the recursion with the head of the queue, this way even if it's over maxWeight it'll return a single element head else { - val (head, tail) = takeWhileWeightedRec(queue.tail, queue.headOption.toVector, queue.headOption.map(weightFunction).getOrElse(n.zero)) + val (head, tail) = takeWhileWeightedRec(queue.tail, + queue.headOption.toVector, + queue.headOption.map(weightFunction).getOrElse(n.zero) + ) DeQueued(head, tail) } } } implicit class EnhancedMapLike[A, +B, +This <: Map[A, B]](val mapLike: Map[A, B]) { + /** * 'safe' in that unlike the implementation hiding behind `MapLike#mapValues` this is strict. i.e. this will only * evaluate the supplied function once on each value and at the time this method is called. @@ -104,17 +113,15 @@ object EnhancedCollections { /** * Based on scalaz's intersectWith, applies `f` to values of keys found in this `mapLike` and map */ - def intersectWith[C, D](map: Map[A, C])(f: (B, C) => D): Map[A, D] = { + def intersectWith[C, D](map: Map[A, C])(f: (B, C) => D): Map[A, D] = mapLike collect { case (mapLikeKey, mapLikeValue) if map.contains(mapLikeKey) => mapLikeKey -> f(mapLikeValue, map(mapLikeKey)) } - } } implicit class EnhancedNonEmptyList[A](val nel: NonEmptyList[A]) extends AnyVal { - def foreach(f: A => Unit): Unit = { + def foreach(f: A => Unit): Unit = nel.toList foreach f - } } } diff --git a/common/src/main/scala/common/collections/Table.scala b/common/src/main/scala/common/collections/Table.scala index 57c1b3f4a57..3f8b262bfb1 100644 --- a/common/src/main/scala/common/collections/Table.scala +++ b/common/src/main/scala/common/collections/Table.scala @@ -3,6 +3,7 @@ package common.collections import scala.collection.immutable object Table { + /** * Instantiates an empty table */ @@ -22,7 +23,7 @@ object Table { * @tparam V type of the value */ case class Table[R, C, V](table: Map[R, Map[C, V]]) { - + /** * Returns true if the table contains a value at row / column */ @@ -51,18 +52,16 @@ case class Table[R, C, V](table: Map[R, Map[C, V]]) { /** * Add a value at row / column */ - def add(row: R, column: C, value: V): Table[R, C, V] = { + def add(row: R, column: C, value: V): Table[R, C, V] = this.copy( table = table.updated(row, table.getOrElse(row, Map.empty).updated(column, value)) ) - } /** * Add all values */ - def addAll(values: Iterable[(R, C, V)]): Table[R, C, V] = { + def addAll(values: Iterable[(R, C, V)]): Table[R, C, V] = values.foldLeft(this)(_.addTriplet(_)) - } /** * Add a value as a triplet diff --git a/common/src/main/scala/common/collections/WeightedQueue.scala b/common/src/main/scala/common/collections/WeightedQueue.scala index 1ba5d869a64..fecb857db9b 100644 --- a/common/src/main/scala/common/collections/WeightedQueue.scala +++ b/common/src/main/scala/common/collections/WeightedQueue.scala @@ -4,9 +4,8 @@ import common.collections.EnhancedCollections._ import scala.collection.immutable.Queue object WeightedQueue { - def empty[T, W](weightFunction: T => W)(implicit n: Numeric[W]) = { + def empty[T, W](weightFunction: T => W)(implicit n: Numeric[W]) = WeightedQueue(Queue.empty[T], weightFunction, n.zero) - } } /** @@ -14,27 +13,24 @@ object WeightedQueue { * In addition to the queue, a weight function is provided that provides the weight W of an element T. * The total weight of the queue is accessible, as well as a method to take the head of the queue based on a max weight value. */ -final case class WeightedQueue[T, W](innerQueue: Queue[T], - private val weightFunction: T => W, - weight: W)(implicit n: Numeric[W]) { +final case class WeightedQueue[T, W](innerQueue: Queue[T], private val weightFunction: T => W, weight: W)(implicit + n: Numeric[W] +) { import n._ - def enqueue(element: T): WeightedQueue[T, W] = { + def enqueue(element: T): WeightedQueue[T, W] = this.copy(innerQueue = innerQueue.enqueue(element), weight = weight + weightFunction(element)) - } def dequeue: (T, WeightedQueue[T, W]) = { val (element, tail) = innerQueue.dequeue element -> this.copy(innerQueue = tail, weight = weight - weightFunction(element)) } - def dequeueOption: Option[(T, WeightedQueue[T, W])] = { - innerQueue.dequeueOption map { - case (element, tail) => - element -> this.copy(innerQueue = tail, weight = weight - weightFunction(element)) + def dequeueOption: Option[(T, WeightedQueue[T, W])] = + innerQueue.dequeueOption map { case (element, tail) => + element -> this.copy(innerQueue = tail, weight = weight - weightFunction(element)) } - } - + def behead(maxWeight: W, maxLength: Option[Int] = None, strict: Boolean = false): (Vector[T], WeightedQueue[T, W]) = { val DeQueued(head, tail) = innerQueue.takeWhileWeighted(maxWeight, weightFunction, maxLength, strict) head -> this.copy(innerQueue = tail, weight = weight - head.map(weightFunction).sum) diff --git a/common/src/main/scala/common/exception/ExceptionAggregation.scala b/common/src/main/scala/common/exception/ExceptionAggregation.scala index bd7f030331c..94059e94f50 100644 --- a/common/src/main/scala/common/exception/ExceptionAggregation.scala +++ b/common/src/main/scala/common/exception/ExceptionAggregation.scala @@ -8,12 +8,11 @@ import common.exception.Aggregation._ import scala.annotation.tailrec object Aggregation { - def formatMessageWithList(message: String, list: Iterable[String]) = { + def formatMessageWithList(message: String, list: Iterable[String]) = if (list.nonEmpty) { val messages = s"\n${list.mkString("\n")}" s"$message:$messages" } else message - } def flattenThrowable(throwable: Throwable) = { @tailrec @@ -57,12 +56,12 @@ trait ThrowableAggregation extends MessageAggregation { override def errorMessages = throwables map buildMessage private def buildMessage(t: Throwable): String = t match { - // The message for file not found exception only contains the file name, so add the actual reason + // The message for file not found exception only contains the file name, so add the actual reason case _: FileNotFoundException | _: NoSuchFileException => s"File not found ${t.getMessage}" case aggregation: ThrowableAggregation => formatMessageWithList(aggregation.exceptionContext, aggregation.throwables.map(buildMessage).map("\t" + _)) case other => - val cause = Option(other.getCause) map { c => s"\n\t${buildMessage(c)}" } getOrElse "" + val cause = Option(other.getCause) map { c => s"\n\t${buildMessage(c)}" } getOrElse "" s"${other.getMessage}$cause" } } @@ -70,6 +69,14 @@ trait ThrowableAggregation extends MessageAggregation { /** * Generic convenience case class for aggregated exceptions. */ -case class AggregatedException(exceptionContext: String, throwables: Iterable[Throwable]) extends Exception with ThrowableAggregation -case class AggregatedMessageException(exceptionContext: String, errorMessages: Iterable[String]) extends Exception with MessageAggregation -case class CompositeException(exceptionContext: String, throwables: Iterable[Throwable], override val errorMessages: Iterable[String]) extends Exception with ThrowableAggregation +case class AggregatedException(exceptionContext: String, throwables: Iterable[Throwable]) + extends Exception + with ThrowableAggregation +case class AggregatedMessageException(exceptionContext: String, errorMessages: Iterable[String]) + extends Exception + with MessageAggregation +case class CompositeException(exceptionContext: String, + throwables: Iterable[Throwable], + override val errorMessages: Iterable[String] +) extends Exception + with ThrowableAggregation diff --git a/common/src/main/scala/common/exception/package.scala b/common/src/main/scala/common/exception/package.scala index dccd49f2643..48d3dd43fb3 100644 --- a/common/src/main/scala/common/exception/package.scala +++ b/common/src/main/scala/common/exception/package.scala @@ -4,7 +4,6 @@ import cats.effect.IO package object exception { - def toIO[A](option: Option[A], errorMsg: String): IO[A] = { + def toIO[A](option: Option[A], errorMsg: String): IO[A] = IO.fromEither(option.toRight(new RuntimeException(errorMsg))) - } } diff --git a/common/src/main/scala/common/numeric/IntegerUtil.scala b/common/src/main/scala/common/numeric/IntegerUtil.scala index 4c33c5ace64..e24e478d400 100644 --- a/common/src/main/scala/common/numeric/IntegerUtil.scala +++ b/common/src/main/scala/common/numeric/IntegerUtil.scala @@ -2,14 +2,13 @@ package common.numeric object IntegerUtil { - private def ordinal(int: Int): String = { + private def ordinal(int: Int): String = int match { case 1 => "st" case 2 => "nd" case 3 => "rd" case _ => "th" } - } implicit class IntEnhanced(val value: Int) extends AnyVal { def toOrdinal: String = value match { @@ -19,9 +18,8 @@ object IntegerUtil { s"$v$suffix" } - def isBetweenInclusive(min: Int, max: Int): Boolean = { + def isBetweenInclusive(min: Int, max: Int): Boolean = min <= value && value <= max - } } } diff --git a/common/src/main/scala/common/transforms/package.scala b/common/src/main/scala/common/transforms/package.scala index 028f426d54a..112fb22d682 100644 --- a/common/src/main/scala/common/transforms/package.scala +++ b/common/src/main/scala/common/transforms/package.scala @@ -12,30 +12,42 @@ package object transforms { object CheckedAtoB { def apply[A, B](implicit runner: CheckedAtoB[A, B]): CheckedAtoB[A, B] = runner def fromCheck[A, B](run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli(run) - def fromCheck[A, B](context: String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli(runCheckWithContext(run, _ => context)) - def fromCheck[A, B](context: A => String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli(runCheckWithContext(run, context)) + def fromCheck[A, B](context: String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli( + runCheckWithContext(run, _ => context) + ) + def fromCheck[A, B](context: A => String)(run: A => Checked[B]): CheckedAtoB[A, B] = Kleisli( + runCheckWithContext(run, context) + ) def fromErrorOr[A, B](run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli(runThenCheck(run)) - def fromErrorOr[A, B](context: String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli(runErrorOrWithContext(run, _ => context)) - def fromErrorOr[A, B](context: A => String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli(runErrorOrWithContext(run, context)) - private def runThenCheck[A, B](run: A => ErrorOr[B]): A => Checked[B] = (a: A) => { run(a).toEither } - private def runErrorOrWithContext[A, B](run: A => ErrorOr[B], context: A => String): A => Checked[B] = (a: A) => { run(a).toEither.contextualizeErrors(context(a)) } - private def runCheckWithContext[A, B](run: A => Checked[B], context: A => String): A => Checked[B] = (a: A) => { run(a).contextualizeErrors(context(a)) } + def fromErrorOr[A, B](context: String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli( + runErrorOrWithContext(run, _ => context) + ) + def fromErrorOr[A, B](context: A => String)(run: A => ErrorOr[B]): CheckedAtoB[A, B] = Kleisli( + runErrorOrWithContext(run, context) + ) + private def runThenCheck[A, B](run: A => ErrorOr[B]): A => Checked[B] = (a: A) => run(a).toEither + private def runErrorOrWithContext[A, B](run: A => ErrorOr[B], context: A => String): A => Checked[B] = (a: A) => + run(a).toEither.contextualizeErrors(context(a)) + private def runCheckWithContext[A, B](run: A => Checked[B], context: A => String): A => Checked[B] = (a: A) => + run(a).contextualizeErrors(context(a)) - def firstSuccess[A, B](options: List[CheckedAtoB[A, B]], operationName: String): CheckedAtoB[A, B] = Kleisli[Checked, A, B] { a => - if (options.isEmpty) { - s"Unable to $operationName: No import resolvers provided".invalidNelCheck - } else { - val firstAttempt = options.head.run(a) - options.tail.foldLeft[Checked[B]](firstAttempt) { (currentResult, nextOption) => - currentResult match { - case v: Right[_, _] => v - case Left(currentErrors) => nextOption.run(a) match { + def firstSuccess[A, B](options: List[CheckedAtoB[A, B]], operationName: String): CheckedAtoB[A, B] = + Kleisli[Checked, A, B] { a => + if (options.isEmpty) { + s"Unable to $operationName: No import resolvers provided".invalidNelCheck + } else { + val firstAttempt = options.head.run(a) + options.tail.foldLeft[Checked[B]](firstAttempt) { (currentResult, nextOption) => + currentResult match { case v: Right[_, _] => v - case Left(newErrors) => Left(currentErrors ++ newErrors.toList) + case Left(currentErrors) => + nextOption.run(a) match { + case v: Right[_, _] => v + case Left(newErrors) => Left(currentErrors ++ newErrors.toList) + } } } } } - } } } diff --git a/common/src/main/scala/common/util/Backoff.scala b/common/src/main/scala/common/util/Backoff.scala index f748785eb10..747508083aa 100644 --- a/common/src/main/scala/common/util/Backoff.scala +++ b/common/src/main/scala/common/util/Backoff.scala @@ -3,8 +3,10 @@ package common.util import scala.concurrent.duration.FiniteDuration trait Backoff { + /** Next interval in millis */ def backoffMillis: Long + /** Get the next instance of backoff. This should be called after every call to backoffMillis */ def next: Backoff } diff --git a/common/src/main/scala/common/util/IORetry.scala b/common/src/main/scala/common/util/IORetry.scala index a8a7a63a209..db983fa7711 100644 --- a/common/src/main/scala/common/util/IORetry.scala +++ b/common/src/main/scala/common/util/IORetry.scala @@ -7,7 +7,7 @@ import scala.util.control.NonFatal object IORetry { def noOpOnRetry[S]: (Throwable, S) => S = (_, s) => s - + object StatefulIoError { def noop[S] = new StatefulIoError[S] { override def toThrowable(state: S, throwable: Throwable) = throwable @@ -35,8 +35,8 @@ object IORetry { backoff: Backoff, isRetryable: Throwable => Boolean = throwableToTrue, isInfinitelyRetryable: Throwable => Boolean = throwableToFalse, - onRetry: (Throwable, S) => S = noOpOnRetry[S]) - (implicit timer: Timer[IO], statefulIoException: StatefulIoError[S]): IO[A] = { + onRetry: (Throwable, S) => S = noOpOnRetry[S] + )(implicit timer: Timer[IO], statefulIoException: StatefulIoError[S]): IO[A] = { lazy val delay = backoff.backoffMillis.millis def fail(throwable: Throwable) = IO.raiseError(statefulIoException.toThrowable(state, throwable)) @@ -49,10 +49,16 @@ object IORetry { if (retriesLeft.forall(_ > 0)) { for { _ <- IO.sleep(delay) - retried <- withRetry(io, onRetry(throwable, state), retriesLeft, backoff.next, isRetryable, isInfinitelyRetryable, onRetry) + retried <- withRetry(io, + onRetry(throwable, state), + retriesLeft, + backoff.next, + isRetryable, + isInfinitelyRetryable, + onRetry + ) } yield retried - } - else fail(throwable) + } else fail(throwable) case fatal => throw fatal } diff --git a/common/src/main/scala/common/util/IntrospectableLazy.scala b/common/src/main/scala/common/util/IntrospectableLazy.scala index e578f49c1a1..7fe05dd1f4c 100644 --- a/common/src/main/scala/common/util/IntrospectableLazy.scala +++ b/common/src/main/scala/common/util/IntrospectableLazy.scala @@ -21,20 +21,21 @@ object IntrospectableLazy { } -class IntrospectableLazy[A] private(f: => A) { +class IntrospectableLazy[A] private (f: => A) { private var option: Option[A] = None - def apply(): A = { + def apply(): A = option match { case Some(a) => a case None => - synchronized { option match { - case Some(a) => a - case None => val a = f; option = Some(a); a - }} + synchronized { + option match { + case Some(a) => a + case None => val a = f; option = Some(a); a + } + } } - } def exists: Boolean = option.isDefined diff --git a/common/src/main/scala/common/util/StringUtil.scala b/common/src/main/scala/common/util/StringUtil.scala index 044ef890dea..2ddce884112 100644 --- a/common/src/main/scala/common/util/StringUtil.scala +++ b/common/src/main/scala/common/util/StringUtil.scala @@ -64,11 +64,10 @@ object StringUtil { */ def relativeDirectory: String = string.ensureNoLeadingSlash.ensureSlashed - def elided(limit: Int): String = { + def elided(limit: Int): String = if (string.length > limit) { s"(elided) ${string.take(limit)}..." } else string - } /** * Removes userInfo and sensitive query parts from strings that are RFC 2396 URIs. @@ -84,10 +83,9 @@ object StringUtil { * - the StringUtilSpec for current expectations * - https://stackoverflow.com/questions/4571346/how-to-encode-url-to-avoid-special-characters-in-java#answer-4571518 */ - def maskSensitiveUri: String = { + def maskSensitiveUri: String = Try(new URI(string)) .map(_.maskSensitive.toASCIIString) .getOrElse(string) - } } } diff --git a/common/src/main/scala/common/util/TerminalUtil.scala b/common/src/main/scala/common/util/TerminalUtil.scala index dab7fb9ce17..33e30f612df 100644 --- a/common/src/main/scala/common/util/TerminalUtil.scala +++ b/common/src/main/scala/common/util/TerminalUtil.scala @@ -1,15 +1,15 @@ package common.util object TerminalUtil { - def highlight(colorCode:Int, string:String) = s"\u001B[38;5;${colorCode}m$string\u001B[0m" + def highlight(colorCode: Int, string: String) = s"\u001B[38;5;${colorCode}m$string\u001B[0m" def mdTable(rows: Seq[Seq[String]], header: Seq[String]): String = { - def maxWidth(lengths: Seq[Seq[Int]], column: Int) = lengths.map { length => length(column) }.max - val widths = (rows :+ header).map { row => row.map { s => s.length } } - val maxWidths = widths.head.indices.map { column => maxWidth(widths, column) } - val tableHeader = header.indices.map { i => header(i).padTo(maxWidths(i), " ").mkString("") }.mkString("|") - val tableDivider = header.indices.map { i => "-" * maxWidths(i) }.mkString("|") + def maxWidth(lengths: Seq[Seq[Int]], column: Int) = lengths.map(length => length(column)).max + val widths = (rows :+ header).map(row => row.map(s => s.length)) + val maxWidths = widths.head.indices.map(column => maxWidth(widths, column)) + val tableHeader = header.indices.map(i => header(i).padTo(maxWidths(i), ' ').mkString("")).mkString("|") + val tableDivider = header.indices.map(i => "-" * maxWidths(i)).mkString("|") val tableRows = rows.map { row => - val mdRow = row.indices.map { i => row(i).padTo(maxWidths(i), " ").mkString("") }.mkString("|") + val mdRow = row.indices.map(i => row(i).padTo(maxWidths(i), ' ').mkString("")).mkString("|") s"|$mdRow|" } s"|$tableHeader|\n|$tableDivider|\n${tableRows.mkString("\n")}\n" diff --git a/common/src/main/scala/common/util/TimeUtil.scala b/common/src/main/scala/common/util/TimeUtil.scala index dbd94c99b80..8c64dd341ce 100644 --- a/common/src/main/scala/common/util/TimeUtil.scala +++ b/common/src/main/scala/common/util/TimeUtil.scala @@ -4,6 +4,7 @@ import java.time.format.DateTimeFormatter import java.time.{OffsetDateTime, ZoneOffset} object TimeUtil { + /** * Instead of "one of" the valid ISO-8601 formats, standardize on this one: * https://github.com/openjdk/jdk/blob/jdk8-b120/jdk/src/share/classes/java/time/OffsetDateTime.java#L1886 @@ -11,12 +12,15 @@ object TimeUtil { private val Iso8601MillisecondsFormat = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSXXXXX") implicit class EnhancedOffsetDateTime(val offsetDateTime: OffsetDateTime) extends AnyVal { + /** * Discards the original timezone and shifts the time to UTC, then returns the ISO-8601 formatted string with * exactly three digits of milliseconds. */ - def toUtcMilliString: String = Option(offsetDateTime).map( - _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) - ).orNull + def toUtcMilliString: String = Option(offsetDateTime) + .map( + _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) + ) + .orNull } } diff --git a/common/src/main/scala/common/util/TryUtil.scala b/common/src/main/scala/common/util/TryUtil.scala index e8edf01f8e5..02a90a5e983 100644 --- a/common/src/main/scala/common/util/TryUtil.scala +++ b/common/src/main/scala/common/util/TryUtil.scala @@ -22,14 +22,13 @@ object TryUtil { def stringifyFailures[T](possibleFailures: Iterable[Try[T]]): Iterable[String] = possibleFailures.collect { case failure: Failure[T] => stringifyFailure(failure) } - private def sequenceIterable[T](tries: Iterable[Try[_]], unbox: () => T, prefixErrorMessage: String): Try[T] = { + private def sequenceIterable[T](tries: Iterable[Try[_]], unbox: () => T, prefixErrorMessage: String): Try[T] = tries collect { case f: Failure[_] => f } match { case failures if failures.nonEmpty => val exceptions = failures.toSeq.map(_.exception) Failure(AggregatedException(prefixErrorMessage, exceptions.toList)) case _ => Success(unbox()) } - } def sequence[T](tries: Seq[Try[T]], prefixErrorMessage: String = ""): Try[Seq[T]] = { def unbox = tries map { _.get } diff --git a/common/src/main/scala/common/util/UriUtil.scala b/common/src/main/scala/common/util/UriUtil.scala index eb8c125768c..b2e4a07bced 100644 --- a/common/src/main/scala/common/util/UriUtil.scala +++ b/common/src/main/scala/common/util/UriUtil.scala @@ -15,20 +15,19 @@ object UriUtil { * - the StringUtilSpec for current expectations * - https://stackoverflow.com/questions/4571346/how-to-encode-url-to-avoid-special-characters-in-java#answer-4571518 */ - def maskSensitive: URI = { + def maskSensitive: URI = Try { - new URI( - uri.getScheme, - null, // Remove all userInfo - uri.getHost, - uri.getPort, - uri.getPath, - Option(uri.getQuery).map(maskSensitiveQuery).orNull, - uri.getFragment, - ) + new URI( + uri.getScheme, + null, // Remove all userInfo + uri.getHost, + uri.getPort, + uri.getPath, + Option(uri.getQuery).map(maskSensitiveQuery).orNull, + uri.getFragment + ) } - .getOrElse(uri) - } + .getOrElse(uri) } private def maskSensitiveQuery(query: String): String = { @@ -85,11 +84,16 @@ object UriUtil { private val SensitiveKeyParts = List( "credential", - "signature", + "signature" + ) + + private val SensitiveKeys = + List( + "sig" ) private def isSensitiveKey(name: String): Boolean = { val lower = name.toLowerCase - SensitiveKeyParts.exists(lower.contains(_)) + SensitiveKeyParts.exists(lower.contains(_)) || SensitiveKeys.exists(lower.equals(_)) } } diff --git a/common/src/main/scala/common/util/VersionUtil.scala b/common/src/main/scala/common/util/VersionUtil.scala index 3ddea0750d5..fcbffca77a4 100644 --- a/common/src/main/scala/common/util/VersionUtil.scala +++ b/common/src/main/scala/common/util/VersionUtil.scala @@ -35,19 +35,17 @@ object VersionUtil { * @param default What to return when the version cannot be found. The parameter passed is the `projectName`. * @return The version from the conf or the default */ - def getVersion(projectName: String, default: String => String = defaultMessage): String = { + def getVersion(projectName: String, default: String => String = defaultMessage): String = ConfigFactory .load(versionConf(projectName)) .as[Option[String]](versionProperty(projectName)) .getOrElse(default(projectName)) - } /** * Instead of returning a version, states that the version conf will be generated by sbt. */ - def defaultMessage(projectName: String): String = { + def defaultMessage(projectName: String): String = s"${versionConf(projectName)}-to-be-generated-by-sbt" - } /** * A regex compatible with the dependency constants in project/Dependencies.scala. @@ -62,7 +60,7 @@ object VersionUtil { * @return The dependency version from project/Dependencies.scala * @throws RuntimeException If the dependency cannot be found */ - def sbtDependencyVersion(dependencyName: String)(projectName: String): String = { + def sbtDependencyVersion(dependencyName: String)(projectName: String): String = try { val dependencies = Paths.get("project/Dependencies.scala").toAbsolutePath val lines = Files.readAllLines(dependencies).asScala @@ -79,6 +77,5 @@ object VersionUtil { e ) } - } } diff --git a/common/src/main/scala/common/validation/ErrorOr.scala b/common/src/main/scala/common/validation/ErrorOr.scala index 4be2aa633c9..3afdfcc5fb7 100644 --- a/common/src/main/scala/common/validation/ErrorOr.scala +++ b/common/src/main/scala/common/validation/ErrorOr.scala @@ -12,9 +12,8 @@ import scala.util.Try object ErrorOr { type ErrorOr[+A] = Validated[NonEmptyList[String], A] - def apply[A](f: => A): ErrorOr[A] = { + def apply[A](f: => A): ErrorOr[A] = Try(f).toErrorOr - } implicit class EnhancedErrorOr[A](val eoa: ErrorOr[A]) extends AnyVal { def contextualizeErrors(s: => String): ErrorOr[A] = eoa.leftMap { errors => @@ -31,17 +30,17 @@ object ErrorOr { } implicit class ShortCircuitingFlatMap[A](val fa: ErrorOr[A]) extends AnyVal { + /** * Not consistent with `Applicative#ap` but useful in for comprehensions. * * @see http://typelevel.org/cats/tut/validated.html#of-flatmaps-and-xors */ - def flatMap[B](f: A => ErrorOr[B]): ErrorOr[B] = { + def flatMap[B](f: A => ErrorOr[B]): ErrorOr[B] = fa match { case Valid(v) => ErrorOr(f(v)).flatten case i @ Invalid(_) => i } - } } implicit class MapErrorOrRhs[A, B](val m: Map[A, ErrorOr[B]]) extends AnyVal { @@ -49,8 +48,8 @@ object ErrorOr { } implicit class MapTraversal[A, B](val m: Map[A, B]) extends AnyVal { - def traverse[C,D](f: ((A,B)) => ErrorOr[(C,D)]): ErrorOr[Map[C,D]] = m.toList.traverse(f).map(_.toMap) - def traverseValues[C](f: B => ErrorOr[C]): ErrorOr[Map[A,C]] = m.traverse { case (a, b) => f(b).map(c => (a,c)) } + def traverse[C, D](f: ((A, B)) => ErrorOr[(C, D)]): ErrorOr[Map[C, D]] = m.toList.traverse(f).map(_.toMap) + def traverseValues[C](f: B => ErrorOr[C]): ErrorOr[Map[A, C]] = m.traverse { case (a, b) => f(b).map(c => (a, c)) } } // Note! See the bottom of this file for a generator function for 2 through 22 of these near-identical ShortCircuitingFlatMapTupleNs... @@ -67,83 +66,346 @@ object ErrorOr { def flatMapN[T_OUT](f3: (A, B, C) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t3.tupled flatMap f3.tupled } - implicit class ShortCircuitingFlatMapTuple4[A, B, C, D](val t4: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple4[A, B, C, D](val t4: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D])) + extends AnyVal { def flatMapN[T_OUT](f4: (A, B, C, D) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t4.tupled flatMap f4.tupled } - implicit class ShortCircuitingFlatMapTuple5[A, B, C, D, E](val t5: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple5[A, B, C, D, E]( + val t5: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E]) + ) extends AnyVal { def flatMapN[T_OUT](f5: (A, B, C, D, E) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t5.tupled flatMap f5.tupled } - implicit class ShortCircuitingFlatMapTuple6[A, B, C, D, E, F](val t6: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple6[A, B, C, D, E, F]( + val t6: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F]) + ) extends AnyVal { def flatMapN[T_OUT](f6: (A, B, C, D, E, F) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t6.tupled flatMap f6.tupled } - implicit class ShortCircuitingFlatMapTuple7[A, B, C, D, E, F, G](val t7: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple7[A, B, C, D, E, F, G]( + val t7: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G]) + ) extends AnyVal { def flatMapN[T_OUT](f7: (A, B, C, D, E, F, G) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t7.tupled flatMap f7.tupled } - implicit class ShortCircuitingFlatMapTuple8[A, B, C, D, E, F, G, H](val t8: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple8[A, B, C, D, E, F, G, H]( + val t8: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H]) + ) extends AnyVal { def flatMapN[T_OUT](f8: (A, B, C, D, E, F, G, H) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t8.tupled flatMap f8.tupled } - implicit class ShortCircuitingFlatMapTuple9[A, B, C, D, E, F, G, H, I](val t9: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I])) extends AnyVal { + implicit class ShortCircuitingFlatMapTuple9[A, B, C, D, E, F, G, H, I]( + val t9: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I]) + ) extends AnyVal { def flatMapN[T_OUT](f9: (A, B, C, D, E, F, G, H, I) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t9.tupled flatMap f9.tupled } - implicit class ShortCircuitingFlatMapTuple10[A, B, C, D, E, F, G, H, I, J](val t10: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J])) extends AnyVal { - def flatMapN[T_OUT](f10: (A, B, C, D, E, F, G, H, I, J) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t10.tupled flatMap f10.tupled + implicit class ShortCircuitingFlatMapTuple10[A, B, C, D, E, F, G, H, I, J]( + val t10: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f10: (A, B, C, D, E, F, G, H, I, J) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t10.tupled flatMap f10.tupled } - implicit class ShortCircuitingFlatMapTuple11[A, B, C, D, E, F, G, H, I, J, K](val t11: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K])) extends AnyVal { - def flatMapN[T_OUT](f11: (A, B, C, D, E, F, G, H, I, J, K) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t11.tupled flatMap f11.tupled + implicit class ShortCircuitingFlatMapTuple11[A, B, C, D, E, F, G, H, I, J, K]( + val t11: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f11: (A, B, C, D, E, F, G, H, I, J, K) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t11.tupled flatMap f11.tupled } - implicit class ShortCircuitingFlatMapTuple12[A, B, C, D, E, F, G, H, I, J, K, L](val t12: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L])) extends AnyVal { - def flatMapN[T_OUT](f12: (A, B, C, D, E, F, G, H, I, J, K, L) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t12.tupled flatMap f12.tupled + implicit class ShortCircuitingFlatMapTuple12[A, B, C, D, E, F, G, H, I, J, K, L]( + val t12: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f12: (A, B, C, D, E, F, G, H, I, J, K, L) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t12.tupled flatMap f12.tupled } - implicit class ShortCircuitingFlatMapTuple13[A, B, C, D, E, F, G, H, I, J, K, L, M](val t13: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M])) extends AnyVal { - def flatMapN[T_OUT](f13: (A, B, C, D, E, F, G, H, I, J, K, L, M) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t13.tupled flatMap f13.tupled + implicit class ShortCircuitingFlatMapTuple13[A, B, C, D, E, F, G, H, I, J, K, L, M]( + val t13: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f13: (A, B, C, D, E, F, G, H, I, J, K, L, M) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t13.tupled flatMap f13.tupled } - implicit class ShortCircuitingFlatMapTuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N](val t14: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N])) extends AnyVal { - def flatMapN[T_OUT](f14: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t14.tupled flatMap f14.tupled + implicit class ShortCircuitingFlatMapTuple14[A, B, C, D, E, F, G, H, I, J, K, L, M, N]( + val t14: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f14: (A, B, C, D, E, F, G, H, I, J, K, L, M, N) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t14.tupled flatMap f14.tupled } - implicit class ShortCircuitingFlatMapTuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O](val t15: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O])) extends AnyVal { - def flatMapN[T_OUT](f15: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t15.tupled flatMap f15.tupled + implicit class ShortCircuitingFlatMapTuple15[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O]( + val t15: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f15: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t15.tupled flatMap f15.tupled } - implicit class ShortCircuitingFlatMapTuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P](val t16: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P])) extends AnyVal { - def flatMapN[T_OUT](f16: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t16.tupled flatMap f16.tupled + implicit class ShortCircuitingFlatMapTuple16[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P]( + val t16: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f16: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t16.tupled flatMap f16.tupled } - implicit class ShortCircuitingFlatMapTuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q](val t17: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q])) extends AnyVal { - def flatMapN[T_OUT](f17: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t17.tupled flatMap f17.tupled + implicit class ShortCircuitingFlatMapTuple17[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q]( + val t17: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f17: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t17.tupled flatMap f17.tupled } - implicit class ShortCircuitingFlatMapTuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R](val t18: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R])) extends AnyVal { - def flatMapN[T_OUT](f18: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t18.tupled flatMap f18.tupled + implicit class ShortCircuitingFlatMapTuple18[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R]( + val t18: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R] + ) + ) extends AnyVal { + def flatMapN[T_OUT](f18: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = + t18.tupled flatMap f18.tupled } - implicit class ShortCircuitingFlatMapTuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S](val t19: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S])) extends AnyVal { - def flatMapN[T_OUT](f19: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t19.tupled flatMap f19.tupled + implicit class ShortCircuitingFlatMapTuple19[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S]( + val t19: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f19: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t19.tupled flatMap f19.tupled } - implicit class ShortCircuitingFlatMapTuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T](val t20: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S], ErrorOr[T])) extends AnyVal { - def flatMapN[T_OUT](f20: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t20.tupled flatMap f20.tupled + implicit class ShortCircuitingFlatMapTuple20[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T]( + val t20: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S], + ErrorOr[T] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f20: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t20.tupled flatMap f20.tupled } - implicit class ShortCircuitingFlatMapTuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U](val t21: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S], ErrorOr[T], ErrorOr[U])) extends AnyVal { - def flatMapN[T_OUT](f21: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t21.tupled flatMap f21.tupled + implicit class ShortCircuitingFlatMapTuple21[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U]( + val t21: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S], + ErrorOr[T], + ErrorOr[U] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f21: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t21.tupled flatMap f21.tupled } - implicit class ShortCircuitingFlatMapTuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V](val t22: (ErrorOr[A], ErrorOr[B], ErrorOr[C], ErrorOr[D], ErrorOr[E], ErrorOr[F], ErrorOr[G], ErrorOr[H], ErrorOr[I], ErrorOr[J], ErrorOr[K], ErrorOr[L], ErrorOr[M], ErrorOr[N], ErrorOr[O], ErrorOr[P], ErrorOr[Q], ErrorOr[R], ErrorOr[S], ErrorOr[T], ErrorOr[U], ErrorOr[V])) extends AnyVal { - def flatMapN[T_OUT](f22: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t22.tupled flatMap f22.tupled + implicit class ShortCircuitingFlatMapTuple22[A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V]( + val t22: (ErrorOr[A], + ErrorOr[B], + ErrorOr[C], + ErrorOr[D], + ErrorOr[E], + ErrorOr[F], + ErrorOr[G], + ErrorOr[H], + ErrorOr[I], + ErrorOr[J], + ErrorOr[K], + ErrorOr[L], + ErrorOr[M], + ErrorOr[N], + ErrorOr[O], + ErrorOr[P], + ErrorOr[Q], + ErrorOr[R], + ErrorOr[S], + ErrorOr[T], + ErrorOr[U], + ErrorOr[V] + ) + ) extends AnyVal { + def flatMapN[T_OUT]( + f22: (A, B, C, D, E, F, G, H, I, J, K, L, M, N, O, P, Q, R, S, T, U, V) => ErrorOr[T_OUT] + ): ErrorOr[T_OUT] = t22.tupled flatMap f22.tupled } object ErrorOrGen { + /** * Because maintaining 22 near-identical functions is a bore... * This function can regenerate them if we need to make changes. @@ -168,8 +430,7 @@ object ErrorOr { | ${line2(n)} |} | - |""".stripMargin + |""".stripMargin } } } - diff --git a/common/src/main/scala/common/validation/IOChecked.scala b/common/src/main/scala/common/validation/IOChecked.scala index f838e5ceb32..62e61f87eb4 100644 --- a/common/src/main/scala/common/validation/IOChecked.scala +++ b/common/src/main/scala/common/validation/IOChecked.scala @@ -1,6 +1,5 @@ package common.validation - import cats.arrow.FunctionK import cats.data.EitherT.fromEither import cats.data.{EitherT, NonEmptyList, ValidatedNel} @@ -22,6 +21,7 @@ object IOChecked { * The monad transformer allows to flatMap over the value while keeping the IO effect as well as the list of potential errors */ type IOChecked[A] = EitherT[IO, NonEmptyList[String], A] + /** * Fixes the left type of Either to Throwable * This is useful when calling on IO[A].attempt, transforming it to IO[ Either[Throwable, A] ] == IO[ Attempt[A] ] @@ -40,22 +40,22 @@ object IOChecked { */ implicit val eitherThrowableApplicative = new Applicative[Attempt] { override def pure[A](x: A) = Right(x) - override def ap[A, B](ff: Attempt[A => B])(fa: Attempt[A]): Attempt[B] = { + override def ap[A, B](ff: Attempt[A => B])(fa: Attempt[A]): Attempt[B] = (fa, ff) match { - // Both have a list or error messages, combine them in a single one - case (Left(t1: MessageAggregation), Left(t2: MessageAggregation)) => Left(AggregatedMessageException("", t1.errorMessages ++ t2.errorMessages)) - // Only one of them is a MessageAggregation, combined the errors and the other exception in a CompositeException + // Both have a list or error messages, combine them in a single one + case (Left(t1: MessageAggregation), Left(t2: MessageAggregation)) => + Left(AggregatedMessageException("", t1.errorMessages ++ t2.errorMessages)) + // Only one of them is a MessageAggregation, combined the errors and the other exception in a CompositeException case (Left(t1: MessageAggregation), Left(t2)) => Left(CompositeException("", List(t2), t1.errorMessages)) case (Left(t2), Left(t1: MessageAggregation)) => Left(CompositeException("", List(t2), t1.errorMessages)) - // None of them is a MessageAggregation, combine the 2 throwables in an AggregatedException + // None of them is a MessageAggregation, combine the 2 throwables in an AggregatedException case (Left(t1), Left(t2)) => Left(AggregatedException("", List(t1, t2))) - // Success case, apply f on v + // Success case, apply f on v case (Right(v), Right(f)) => Right(f(v)) - // Default failure case, just keep the failure + // Default failure case, just keep the failure case (Left(t1), _) => Left(t1) case (_, Left(t1)) => Left(t1) } - } } /** @@ -100,7 +100,7 @@ object IOChecked { * We now have an IO[ Either[NonEmptyList[String], A] ] which we can wrap into an IOChecked with EitherT.apply */ override def sequential: FunctionK[IOCheckedPar, IOChecked] = - new FunctionK[IOCheckedPar, IOChecked] { + new FunctionK[IOCheckedPar, IOChecked] { def apply[A](fa: IOCheckedPar[A]): IOChecked[A] = EitherT { IO.Par.unwrap(fa: IO.Par[Attempt[A]]) flatMap { case Left(t: MessageAggregation) => IO.pure(Left(NonEmptyList.fromListUnsafe(t.errorMessages.toList))) @@ -138,7 +138,7 @@ object IOChecked { def error[A](error: String, tail: String*): IOChecked[A] = EitherT.leftT { NonEmptyList.of(error, tail: _*) } - + def pure[A](value: A): IOChecked[A] = EitherT.pure(value) def goIOChecked[A](f: => A): IOChecked[A] = Try(f).toIOChecked @@ -150,12 +150,11 @@ object IOChecked { implicit class EnhancedIOChecked[A](val p: IOChecked[A]) extends AnyVal { import cats.syntax.either._ - def toChecked: Checked[A] = { + def toChecked: Checked[A] = Try(p.value.unsafeRunSync()) match { case Success(r) => r case Failure(f) => NonEmptyList.one(f.getMessage).asLeft } - } def toErrorOr: ErrorOr[A] = toChecked.toValidated @@ -163,12 +162,11 @@ object IOChecked { def unsafe(context: String) = unsafeToEither().unsafe(context) - def contextualizeErrors(context: String): IOChecked[A] = { - p.leftMap({ errors => + def contextualizeErrors(context: String): IOChecked[A] = + p.leftMap { errors => val total = errors.size errors.zipWithIndex map { case (e, i) => s"Failed to $context (reason ${i + 1} of $total): $e" } - }) - } + } } implicit class TryIOChecked[A](val t: Try[A]) extends AnyVal { @@ -176,9 +174,8 @@ object IOChecked { } implicit class FutureIOChecked[A](val future: Future[A]) extends AnyVal { - def toIOChecked(implicit cs: ContextShift[IO]): IOChecked[A] = { + def toIOChecked(implicit cs: ContextShift[IO]): IOChecked[A] = IO.fromFuture(IO(future)).to[IOChecked] - } } implicit class ErrorOrIOChecked[A](val e: ErrorOr[A]) extends AnyVal { @@ -198,9 +195,8 @@ object IOChecked { } implicit class OptionIOChecked[A](val o: Option[A]) extends AnyVal { - def toIOChecked(errorMessage: String): IOChecked[A] = { + def toIOChecked(errorMessage: String): IOChecked[A] = EitherT.fromOption(o, NonEmptyList.of(errorMessage)) - } } type IOCheckedValidated[A] = IO[ValidatedNel[String, A]] diff --git a/common/src/main/scala/common/validation/Validation.scala b/common/src/main/scala/common/validation/Validation.scala index 4099eff8ac6..af568909b5b 100644 --- a/common/src/main/scala/common/validation/Validation.scala +++ b/common/src/main/scala/common/validation/Validation.scala @@ -46,12 +46,12 @@ object Validation { case Failure(f) => defaultThrowableToString(f).invalidNel } - implicit class ValidationOps[B,A](val v: ValidatedNel[B, A]) { - //Convert this into a future by folding over the state and returning the corresponding Future terminal state. + implicit class ValidationOps[B, A](val v: ValidatedNel[B, A]) { + // Convert this into a future by folding over the state and returning the corresponding Future terminal state. def toFuture(f: NonEmptyList[B] => Throwable) = - v fold( - //Use f to turn the failure list into a Throwable, then fail a future with it. - //Function composition lets us ignore the actual argument of the error list + v fold ( + // Use f to turn the failure list into a Throwable, then fail a future with it. + // Function composition lets us ignore the actual argument of the error list Future.failed _ compose f, Future.successful ) @@ -59,31 +59,30 @@ object Validation { implicit class TryValidation[A](val t: Try[A]) extends AnyVal { def toErrorOr: ErrorOr[A] = toErrorOr(defaultThrowableToString) - def toErrorOr(throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = { + def toErrorOr(throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = Validated.fromTry(t).leftMap(throwableToStringFunction).toValidatedNel[String, A] - } def toErrorOrWithContext(context: String): ErrorOr[A] = toErrorOrWithContext(context, defaultThrowableToString) - def toErrorOrWithContext(context: String, - throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = toChecked(throwableToStringFunction) - .contextualizeErrors(context) - .leftMap({contextualizedErrors => - if (t.failed.isFailure) { - val errors = new StringWriter - t.failed.get.printStackTrace(new PrintWriter(errors)) - contextualizedErrors.::(s"Stacktrace: ${errors.toString}") - } else contextualizedErrors - }) - .toValidated + def toErrorOrWithContext(context: String, throwableToStringFunction: ThrowableToStringFunction): ErrorOr[A] = + toChecked(throwableToStringFunction) + .contextualizeErrors(context) + .leftMap { contextualizedErrors => + if (t.failed.isFailure) { + val errors = new StringWriter + t.failed.get.printStackTrace(new PrintWriter(errors)) + contextualizedErrors.::(s"Stacktrace: ${errors.toString}") + } else contextualizedErrors + } + .toValidated def toChecked: Checked[A] = toChecked(defaultThrowableToString) - def toChecked(throwableToStringFunction: ThrowableToStringFunction): Checked[A] = { - Either.fromTry(t).leftMap { ex => NonEmptyList.one(throwableToStringFunction(ex)) } - } + def toChecked(throwableToStringFunction: ThrowableToStringFunction): Checked[A] = + Either.fromTry(t).leftMap(ex => NonEmptyList.one(throwableToStringFunction(ex))) - def toCheckedWithContext(context: String): Checked[A] = toErrorOrWithContext(context, defaultThrowableToString).toEither - def toCheckedWithContext(context: String, - throwableToStringFunction: ThrowableToStringFunction): Checked[A] = toErrorOrWithContext(context, throwableToStringFunction).toEither + def toCheckedWithContext(context: String): Checked[A] = + toErrorOrWithContext(context, defaultThrowableToString).toEither + def toCheckedWithContext(context: String, throwableToStringFunction: ThrowableToStringFunction): Checked[A] = + toErrorOrWithContext(context, throwableToStringFunction).toEither } implicit class ValidationTry[A](val e: ErrorOr[A]) extends AnyVal { @@ -109,12 +108,10 @@ object Validation { } implicit class OptionValidation[A](val o: Option[A]) extends AnyVal { - def toErrorOr(errorMessage: => String): ErrorOr[A] = { + def toErrorOr(errorMessage: => String): ErrorOr[A] = Validated.fromOption(o, NonEmptyList.of(errorMessage)) - } - def toChecked(errorMessage: => String): Checked[A] = { + def toChecked(errorMessage: => String): Checked[A] = Either.fromOption(o, NonEmptyList.of(errorMessage)) - } } } diff --git a/common/src/test/scala/common/assertion/CaseClassAssertions.scala b/common/src/test/scala/common/assertion/CaseClassAssertions.scala index 4cc191df2f6..f273898de1c 100644 --- a/common/src/test/scala/common/assertion/CaseClassAssertions.scala +++ b/common/src/test/scala/common/assertion/CaseClassAssertions.scala @@ -5,10 +5,9 @@ import org.scalatest.matchers.should.Matchers object CaseClassAssertions extends Matchers { implicit class ComparableCaseClass[A <: Product](actualA: A) { // Assumes that expectedA and actualA are the same type. If we don't subtype case classes, that should hold... - def shouldEqualFieldwise(expectedA: A): Unit = { + def shouldEqualFieldwise(expectedA: A): Unit = (0 until actualA.productArity) foreach { i => actualA.productElement(i) should be(expectedA.productElement(i)) } - } } } diff --git a/common/src/test/scala/common/assertion/ErrorOrAssertions.scala b/common/src/test/scala/common/assertion/ErrorOrAssertions.scala index 02a5e594bb4..00c3ab4fcfc 100644 --- a/common/src/test/scala/common/assertion/ErrorOrAssertions.scala +++ b/common/src/test/scala/common/assertion/ErrorOrAssertions.scala @@ -6,7 +6,6 @@ import common.validation.ErrorOr.ErrorOr import org.scalatest.Assertion import org.scalatest.matchers.should.Matchers - object ErrorOrAssertions { implicit class ErrorOrWithAssertions[A](errorOr: ErrorOr[A]) extends Matchers { def shouldBeValid(other: A): Assertion = errorOr match { diff --git a/common/src/test/scala/common/collections/EnhancedCollectionsSpec.scala b/common/src/test/scala/common/collections/EnhancedCollectionsSpec.scala index 12dba2e15d9..d2e0a38878c 100644 --- a/common/src/test/scala/common/collections/EnhancedCollectionsSpec.scala +++ b/common/src/test/scala/common/collections/EnhancedCollectionsSpec.scala @@ -11,7 +11,7 @@ class EnhancedCollectionsSpec extends AsyncFlatSpec with Matchers { behavior of "EnhancedCollections" it should "filter a List by type and return a List" in { - val objectList = List("hello", 3, None, "world") + val objectList = List[Any]("hello", 3, None, "world") val stringList = objectList.filterByType[String] stringList should be(List("hello", "world")) @@ -28,14 +28,14 @@ class EnhancedCollectionsSpec extends AsyncFlatSpec with Matchers { } it should "filter a Set by type and return a Set" in { - val objectSet = Set("hello", 3, None, "world") + val objectSet = Set[Any]("hello", 3, None, "world") val intSet: Set[Int] = objectSet.filterByType[Int] intSet should be(Set(3)) } it should "find the first Int in a List" in { - val objectSet = List("hello", 3, None, 4, "world") + val objectSet = List[Any]("hello", 3, None, 4, "world") objectSet.firstByType[Int] should be(Some(3)) } diff --git a/common/src/test/scala/common/collections/TableSpec.scala b/common/src/test/scala/common/collections/TableSpec.scala index 7000e151df4..581fcf466e8 100644 --- a/common/src/test/scala/common/collections/TableSpec.scala +++ b/common/src/test/scala/common/collections/TableSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "Table" @@ -21,10 +20,14 @@ class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "fill a table" in { - Table.fill(List( - ("a", "b", "c"), - ("d", "e", "f") - )).table shouldBe Map( + Table + .fill( + List( + ("a", "b", "c"), + ("d", "e", "f") + ) + ) + .table shouldBe Map( "a" -> Map("b" -> "c"), "d" -> Map("e" -> "f") ) @@ -48,7 +51,7 @@ class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { someTable.rowOptional("a") shouldBe Some(Map("b" -> "c")) someTable.rowOptional("b") shouldBe None } - + it should "implement row" in { someTable.row("a") shouldBe Map("b" -> "c") someTable.row("b") shouldBe empty @@ -87,7 +90,7 @@ class TableSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { it should "implement addTriplet" in { someTable.addTriplet( - ("0", "1", "2") + ("0", "1", "2") ) shouldBe Table( Map( "a" -> Map("b" -> "c"), diff --git a/common/src/test/scala/common/collections/WeightedQueueSpec.scala b/common/src/test/scala/common/collections/WeightedQueueSpec.scala index 5d29b54949e..ba9cd81d908 100644 --- a/common/src/test/scala/common/collections/WeightedQueueSpec.scala +++ b/common/src/test/scala/common/collections/WeightedQueueSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class WeightedQueueSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "WeightedQueue" @@ -39,7 +38,8 @@ class WeightedQueueSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche it should "behead the queue" in { // A queue of strings for which the weight is the number of char in the string val q = WeightedQueue.empty[String, Int](_.length) - val q2 = q.enqueue("hello") + val q2 = q + .enqueue("hello") .enqueue("hola") .enqueue("bonjour") val (head, q3) = q2.behead(10) diff --git a/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala b/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala index cd425b626f0..071b7836ceb 100644 --- a/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala +++ b/common/src/test/scala/common/exception/ExceptionAggregationSpec.scala @@ -7,7 +7,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers - class ExceptionAggregationSpec extends AnyFlatSpecLike with CromwellTimeoutSpec with Matchers { "MessageAggregation" should "aggregate messages" in { diff --git a/common/src/test/scala/common/mock/MockImplicits.scala b/common/src/test/scala/common/mock/MockImplicits.scala index 3e90e183d8e..424dbfbdfde 100644 --- a/common/src/test/scala/common/mock/MockImplicits.scala +++ b/common/src/test/scala/common/mock/MockImplicits.scala @@ -11,31 +11,27 @@ trait MockImplicits { * https://github.com/etorreborre/specs2/commit/6d56660e70980b5958e6c4ed8fd4158bf1cecf70#diff-a2627f56c432e4bc37f36bc56e13852225813aa604918471b61ec2080462d722 */ implicit class MockEnhanced[A](methodCall: A) { - def returns(result: A): OngoingStubbing[A] = { + def returns(result: A): OngoingStubbing[A] = Mockito.when(methodCall).thenReturn(result) - } - def answers(function: Any => A): OngoingStubbing[A] = { - Mockito.when(methodCall) thenAnswer { - invocationOnMock => { - val args = invocationOnMock.getArguments - // The DSL behavior of the below is directly taken with thanks from the link above. - args.size match { - case 0 => - function match { - case function0: Function0[_] => - function0.apply().asInstanceOf[A] - case _ => - function.apply(invocationOnMock.getMock) - } - case 1 => - function(args(0)) - case _ => - function(args) - } + def answers(function: Any => A): OngoingStubbing[A] = + Mockito.when(methodCall) thenAnswer { invocationOnMock => + val args = invocationOnMock.getArguments + // The DSL behavior of the below is directly taken with thanks from the link above. + args.size match { + case 0 => + function match { + case function0: Function0[_] => + function0.apply().asInstanceOf[A] + case _ => + function.apply(invocationOnMock.getMock) + } + case 1 => + function(args(0)) + case _ => + function(args) } } - } def responds(f: Any => A): OngoingStubbing[A] = answers(f) } diff --git a/common/src/test/scala/common/mock/MockSugar.scala b/common/src/test/scala/common/mock/MockSugar.scala index 27a4ae87b55..6a92bd2fd47 100644 --- a/common/src/test/scala/common/mock/MockSugar.scala +++ b/common/src/test/scala/common/mock/MockSugar.scala @@ -2,7 +2,7 @@ package common.mock import org.mockito.{ArgumentCaptor, Mockito} -import scala.reflect.{ClassTag, classTag} +import scala.reflect.{classTag, ClassTag} /** * Yet another scala wrapper around Mockito. @@ -37,12 +37,11 @@ trait MockSugar extends MockImplicits { * * Note: if you run into issues with `mock` then try [[mockWithDefaults]]. */ - def mock[A: ClassTag]: A = { + def mock[A: ClassTag]: A = Mockito.mock( classTag[A].runtimeClass.asInstanceOf[Class[A]], - Mockito.withSettings().defaultAnswer(Mockito.RETURNS_SMART_NULLS), + Mockito.withSettings().defaultAnswer(Mockito.RETURNS_SMART_NULLS) ) - } /** * Creates a mock returning default values instead of Smart Nulls. @@ -56,16 +55,14 @@ trait MockSugar extends MockImplicits { * * An alternative workaround was to use `Mockito.doReturn(retVal).when(mockObj).someMethod`. */ - def mockWithDefaults[A: ClassTag]: A = { + def mockWithDefaults[A: ClassTag]: A = Mockito.mock( classTag[A].runtimeClass.asInstanceOf[Class[A]], - Mockito.withSettings().defaultAnswer(Mockito.RETURNS_DEFAULTS), + Mockito.withSettings().defaultAnswer(Mockito.RETURNS_DEFAULTS) ) - } - def capture[A: ClassTag]: ArgumentCaptor[A] = { + def capture[A: ClassTag]: ArgumentCaptor[A] = ArgumentCaptor.forClass(classTag[A].runtimeClass.asInstanceOf[Class[A]]) - } } object MockSugar extends MockSugar diff --git a/common/src/test/scala/common/numeric/IntegerUtilSpec.scala b/common/src/test/scala/common/numeric/IntegerUtilSpec.scala index b06af539626..48fc9ed3b3b 100644 --- a/common/src/test/scala/common/numeric/IntegerUtilSpec.scala +++ b/common/src/test/scala/common/numeric/IntegerUtilSpec.scala @@ -5,18 +5,33 @@ import common.numeric.IntegerUtil.IntEnhanced import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class IntegerUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { it should "return ordinal String for any Int" in { - val numbers = List(0, 1, 2, 3, 4, - 10, 11, 12, 13, 14, - 20, 21, 22, 23, 24, - 100, 101, 102, 103, 104) map { _.toOrdinal } + val numbers = List(0, 1, 2, 3, 4, 10, 11, 12, 13, 14, 20, 21, 22, 23, 24, 100, 101, 102, 103, 104) map { + _.toOrdinal + } - val expected = List("0th", "1st", "2nd", "3rd", "4th", - "10th", "11th", "12th", "13th", "14th", - "20th", "21st", "22nd", "23rd", "24th", - "100th", "101st", "102nd", "103rd", "104th") + val expected = List("0th", + "1st", + "2nd", + "3rd", + "4th", + "10th", + "11th", + "12th", + "13th", + "14th", + "20th", + "21st", + "22nd", + "23rd", + "24th", + "100th", + "101st", + "102nd", + "103rd", + "104th" + ) numbers should contain theSameElementsInOrderAs expected } diff --git a/common/src/test/scala/common/util/IntrospectableLazySpec.scala b/common/src/test/scala/common/util/IntrospectableLazySpec.scala index 337e193ab8f..91243135ab6 100644 --- a/common/src/test/scala/common/util/IntrospectableLazySpec.scala +++ b/common/src/test/scala/common/util/IntrospectableLazySpec.scala @@ -22,17 +22,19 @@ class IntrospectableLazySpec extends AnyFlatSpec with CromwellTimeoutSpec with M 4 } - val myLazy = lazily { lazyContents } + val myLazy = lazily(lazyContents) assert(lazyInstantiations == 0) assert(!myLazy.exists) // Fails without `synchronized { ... }` Await.result(Future.sequence( - Seq.fill(100)(Future { - myLazy() shouldBe 4 - }) - ), 1.seconds) + Seq.fill(100)(Future { + myLazy() shouldBe 4 + }) + ), + 1.seconds + ) assert(lazyInstantiations == 1) assert(myLazy.exists) diff --git a/common/src/test/scala/common/util/IoRetrySpec.scala b/common/src/test/scala/common/util/IoRetrySpec.scala index 4a5fc582eac..2ec5c56aba4 100644 --- a/common/src/test/scala/common/util/IoRetrySpec.scala +++ b/common/src/test/scala/common/util/IoRetrySpec.scala @@ -9,7 +9,6 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.ExecutionContext import scala.concurrent.duration._ - class IoRetrySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { implicit val timer = IO.timer(ExecutionContext.global) implicit val ioError = new StatefulIoError[Int] { @@ -28,7 +27,8 @@ class IoRetrySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } val incrementOnRetry: (Throwable, Int) => Int = (_, s) => s + 1 - val io = IORetry.withRetry(work, 1, Option(3), backoff = Backoff.staticBackoff(10.millis), onRetry = incrementOnRetry) + val io = + IORetry.withRetry(work, 1, Option(3), backoff = Backoff.staticBackoff(10.millis), onRetry = incrementOnRetry) val statefulException = the[Exception] thrownBy io.unsafeRunSync() statefulException.getCause shouldBe exception statefulException.getMessage shouldBe "Attempted 3 times" diff --git a/common/src/test/scala/common/util/StringUtilSpec.scala b/common/src/test/scala/common/util/StringUtilSpec.scala index 3d7b8db92d1..f71d79819cb 100644 --- a/common/src/test/scala/common/util/StringUtilSpec.scala +++ b/common/src/test/scala/common/util/StringUtilSpec.scala @@ -17,11 +17,13 @@ class StringUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers // With the elided string, we stop processing early and are able to produce a nice, short string without ever // touching the later elements: - fooOfBars.toPrettyElidedString(1000) should be("""Foo( - | bar = "long long list", - | list = List( - | "blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0", - | "blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah...""".stripMargin) + fooOfBars.toPrettyElidedString(1000) should be( + """Foo( + | bar = "long long list", + | list = List( + | "blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0blah0", + | "blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah1blah...""".stripMargin + ) } @@ -36,84 +38,84 @@ class StringUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers ( "mask user info", "https://user:pass@example.com/path/to/file", - "https://example.com/path/to/file", + "https://example.com/path/to/file" ), ( "mask the entire query if no known sensitive query params are found", s"https://example.com/path/to/file?my_new_hidden_param=$InputToBeMasked", - "https://example.com/path/to/file?masked", + "https://example.com/path/to/file?masked" ), ( "mask credential params", s"https://example.com/path/to/file?my_credential_param=$InputToBeMasked&my_other_param=ok", - s"https://example.com/path/to/file?my_credential_param=$OutputMasked&my_other_param=ok", + s"https://example.com/path/to/file?my_credential_param=$OutputMasked&my_other_param=ok" ), ( "mask signature params", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param=ok", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok" ), ( "mask encoded signature params", s"https://example.com/path/to/file?my_sign%61ture_param=$InputToBeMasked&my_other_param=ok", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=ok" ), ( // There is a note in the docs for common.util.StringUtil.EnhancedString.maskSensitiveUri about this behavior "mask uris with encoded parameters", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param=%26%2F%3F", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=&/?", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=&/?" ), ( "mask uris with parameters without values", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param" ), ( "mask uris with parameters values containing equal signs", s"https://example.com/path/to/file?my_signature_param=$InputToBeMasked&my_other_param=with=equal", - s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=with=equal", + s"https://example.com/path/to/file?my_signature_param=$OutputMasked&my_other_param=with=equal" ), ( "not mask the fragment", s"https://example.com?my_signature_param=$InputToBeMasked#nofilter", - s"https://example.com?my_signature_param=$OutputMasked#nofilter", + s"https://example.com?my_signature_param=$OutputMasked#nofilter" ), ( "not mask the port number", s"https://example.com:1234?my_signature_param=$InputToBeMasked", - s"https://example.com:1234?my_signature_param=$OutputMasked", + s"https://example.com:1234?my_signature_param=$OutputMasked" ), ( // via: https://cr.openjdk.java.net/~dfuchs/writeups/updating-uri/ "not mask a RFC 3986 specific uri", s"urn:isbn:096139210?my_credential_param=$InputToBeMasked", - s"urn:isbn:096139210?my_credential_param=$InputToBeMasked", + s"urn:isbn:096139210?my_credential_param=$InputToBeMasked" ), ( // via: https://bvdp-saturn-dev.appspot.com/#workspaces/general-dev-billing-account/DRS%20and%20Signed%20URL%20Development%20-%20Dev/notebooks/launch/drs_signed_url_flow_kids_dev.ipynb "not mask a DRS CIB URI", "drs://dg.F82A1A:371b834f-a896-42e6-b1d1-9fad96891f33", - "drs://dg.F82A1A:371b834f-a896-42e6-b1d1-9fad96891f33", + "drs://dg.F82A1A:371b834f-a896-42e6-b1d1-9fad96891f33" ), ( // via: https://bvdp-saturn-dev.appspot.com/#workspaces/general-dev-billing-account/DRS%20and%20Signed%20URL%20Development%20-%20Dev/notebooks/launch/drs_signed_url_flow_kids_dev.ipynb "mask an AWS Signed URL", s"https://example-redacted-but-not-masked.s3.amazonaws.com/$InputRedacted.CNVs.p.value.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=$InputToBeMasked&X-Amz-Date=20210504T200819Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&user_id=122&username=$InputRedacted&X-Amz-Signature=$InputToBeMasked", - s"https://example-redacted-but-not-masked.s3.amazonaws.com/$InputRedacted.CNVs.p.value.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=$OutputMasked&X-Amz-Date=20210504T200819Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&user_id=122&username=$InputRedacted&X-Amz-Signature=$OutputMasked", + s"https://example-redacted-but-not-masked.s3.amazonaws.com/$InputRedacted.CNVs.p.value.txt?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=$OutputMasked&X-Amz-Date=20210504T200819Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&user_id=122&username=$InputRedacted&X-Amz-Signature=$OutputMasked" ), ( // via: https://bvdp-saturn-dev.appspot.com/#workspaces/general-dev-billing-account/DRS%20and%20Signed%20URL%20Development%20-%20Dev/notebooks/launch/drs_signed_url_flow_bdcat_dev.ipynb "mask a GCS V2 Signed URL", s"https://storage.googleapis.com/$InputRedacted/testfile.txt?GoogleAccessId=$InputRedacted&Expires=1614119022&Signature=$InputToBeMasked&userProject=$InputRedacted", - s"https://storage.googleapis.com/$InputRedacted/testfile.txt?GoogleAccessId=$InputRedacted&Expires=1614119022&Signature=$OutputMasked&userProject=$InputRedacted", + s"https://storage.googleapis.com/$InputRedacted/testfile.txt?GoogleAccessId=$InputRedacted&Expires=1614119022&Signature=$OutputMasked&userProject=$InputRedacted" ), ( // via: gsutil signurl $HOME/.config/gcloud/legacy_credentials/cromwell@broad-dsde-cromwell-dev.iam.gserviceaccount.com/adc.json gs://cloud-cromwell-dev/some/gumby.png "mask a GCS V4 Signed URL", s"https://storage.googleapis.com/cloud-cromwell-dev/some/gumby.png?x-goog-signature=$InputToBeMasked&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-credential=$InputToBeMasked&x-goog-date=20210505T042119Z&x-goog-expires=3600&x-goog-signedheaders=host", - s"https://storage.googleapis.com/cloud-cromwell-dev/some/gumby.png?x-goog-signature=$OutputMasked&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-credential=$OutputMasked&x-goog-date=20210505T042119Z&x-goog-expires=3600&x-goog-signedheaders=host", - ), + s"https://storage.googleapis.com/cloud-cromwell-dev/some/gumby.png?x-goog-signature=$OutputMasked&x-goog-algorithm=GOOG4-RSA-SHA256&x-goog-credential=$OutputMasked&x-goog-date=20210505T042119Z&x-goog-expires=3600&x-goog-signedheaders=host" + ) ) forAll(maskSensitiveUriTests) { (description, input, expected) => @@ -128,7 +130,7 @@ object StringUtilSpec { final case class Foo(bar: String, list: List[Bar]) final class Bar(index: Int) { - private def longLine(i: Int) = "\"" + s"blah$i" * 100 + "\"" + private def longLine(i: Int) = "\"" + s"blah$i" * 100 + "\"" override def toString: String = if (index < 2) { longLine(index) } else { diff --git a/common/src/test/scala/common/util/TerminalUtilSpec.scala b/common/src/test/scala/common/util/TerminalUtilSpec.scala index c8fda6627e4..1c68b9f17fc 100644 --- a/common/src/test/scala/common/util/TerminalUtilSpec.scala +++ b/common/src/test/scala/common/util/TerminalUtilSpec.scala @@ -5,7 +5,6 @@ import common.util.TerminalUtil._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class TerminalUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "TerminalUtil" diff --git a/common/src/test/scala/common/util/TryUtilSpec.scala b/common/src/test/scala/common/util/TryUtilSpec.scala index d7b8739337b..0bb62f9ed33 100644 --- a/common/src/test/scala/common/util/TryUtilSpec.scala +++ b/common/src/test/scala/common/util/TryUtilSpec.scala @@ -9,7 +9,6 @@ import org.scalatest.matchers.should.Matchers import scala.util.{Failure, Success, Try} import org.scalatest.enablers.Emptiness._ - class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "TryUtil" @@ -80,15 +79,15 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence successful keys and successful values" in { - val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Success("success key") -> Success("success value")), "prefix") + val result: Try[Map[String, String]] = + sequenceKeyValues(Map(Success("success key") -> Success("success value")), "prefix") result.isSuccess should be(true) result.get.toList should contain theSameElementsAs Map("success key" -> "success value") } it should "sequence successful keys and failed values" in { - val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Success("success key") -> Failure(new RuntimeException("failed value"))), "prefix") + val result: Try[Map[String, String]] = + sequenceKeyValues(Map(Success("success key") -> Failure(new RuntimeException("failed value"))), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -98,8 +97,8 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence failed keys and successful values" in { - val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Failure(new RuntimeException("failed key")) -> Success("success value")), "prefix") + val result: Try[Map[String, String]] = + sequenceKeyValues(Map(Failure(new RuntimeException("failed key")) -> Success("success value")), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -110,7 +109,9 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { it should "sequence failed keys and failed values" in { val result: Try[Map[String, String]] = sequenceKeyValues( - Map(Failure(new RuntimeException("failed key")) -> Failure(new RuntimeException("failed value"))), "prefix") + Map(Failure(new RuntimeException("failed key")) -> Failure(new RuntimeException("failed value"))), + "prefix" + ) result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -127,8 +128,8 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence a successful key with a failed value" in { - val result: Try[(String, String)] = sequenceTuple( - (Success("success key"), Failure(new RuntimeException("failed value"))), "prefix") + val result: Try[(String, String)] = + sequenceTuple((Success("success key"), Failure(new RuntimeException("failed value"))), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -138,8 +139,8 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence a failed key with a successful value" in { - val result: Try[(String, String)] = sequenceTuple( - (Failure(new RuntimeException("failed key")), Success("success value")), "prefix") + val result: Try[(String, String)] = + sequenceTuple((Failure(new RuntimeException("failed key")), Success("success value")), "prefix") result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] @@ -149,8 +150,10 @@ class TryUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "sequence a failed key with a failed value" in { - val result: Try[(String, String)] = sequenceTuple( - (Failure(new RuntimeException("failed key")), Failure(new RuntimeException("failed value"))), "prefix") + val result: Try[(String, String)] = + sequenceTuple((Failure(new RuntimeException("failed key")), Failure(new RuntimeException("failed value"))), + "prefix" + ) result.isFailure should be(true) result.failed.get should be(an[AggregatedException]) val exception = result.failed.get.asInstanceOf[AggregatedException] diff --git a/common/src/test/scala/common/util/VersionUtilSpec.scala b/common/src/test/scala/common/util/VersionUtilSpec.scala index 50aab84bad6..5d58d2f0852 100644 --- a/common/src/test/scala/common/util/VersionUtilSpec.scala +++ b/common/src/test/scala/common/util/VersionUtilSpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class VersionUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "VersionUtil" @@ -22,12 +21,12 @@ class VersionUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers it should "getVersion with the default" in { val version = VersionUtil.getVersion("made-up-artifact") - version should be ("made-up-artifact-version.conf-to-be-generated-by-sbt") + version should be("made-up-artifact-version.conf-to-be-generated-by-sbt") } it should "getVersion with a default override" in { val version = VersionUtil.getVersion("made-up-artifact", _ => "default override") - version should be ("default override") + version should be("default override") } it should "defaultMessage" in { @@ -38,7 +37,7 @@ class VersionUtilSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers val expected = intercept[RuntimeException](VersionUtil.sbtDependencyVersion("madeUp")("made-up-project")) expected.getMessage should fullyMatch regex "Did not parse a version for 'madeUpV' from .*/project/Dependencies.scala " + - "\\(This occurred after made-up-project-version.conf was not found.\\)" + "\\(This occurred after made-up-project-version.conf was not found.\\)" } it should "pass sbtDependencyVersion check for typesafeConfig" in { diff --git a/common/src/test/scala/common/validation/CheckedSpec.scala b/common/src/test/scala/common/validation/CheckedSpec.scala index c9b5a387666..23281fccc0b 100644 --- a/common/src/test/scala/common/validation/CheckedSpec.scala +++ b/common/src/test/scala/common/validation/CheckedSpec.scala @@ -6,10 +6,9 @@ import common.validation.Checked._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class CheckedSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "Checked" - + it should "provide helper methods" in { 5.validNelCheck shouldBe Right(5) "argh".invalidNelCheck[Int] shouldBe Left(NonEmptyList.one("argh")) diff --git a/common/src/test/scala/common/validation/ErrorOrSpec.scala b/common/src/test/scala/common/validation/ErrorOrSpec.scala index ac0ebfd957b..55a7d004cc3 100644 --- a/common/src/test/scala/common/validation/ErrorOrSpec.scala +++ b/common/src/test/scala/common/validation/ErrorOrSpec.scala @@ -8,7 +8,6 @@ import common.validation.ErrorOr._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "ErrorOr" @@ -44,11 +43,33 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } val DivBy0Error: String = "Divide by 0!" - def errorOrDiv(v1: Int, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } else { DivBy0Error.invalidNel } - def errorOrDiv(v1: Double, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } else { DivBy0Error.invalidNel } - def errorOrSelect(v1: Int, v2: Int, v3: Int, v4: Int, v5: Int, v6: Int, v7: Int, - v8: Int, v9: Int, v10: Int, v11: Int, v12: Int, v13: Int, v14: Int, - v15: Int, v16: Int, v17: Int, v18: Int, v19: Int, v20: Int, v21: Int, v22: Int): ErrorOr[Int] = Valid(v4 + v6 + v22) + def errorOrDiv(v1: Int, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } + else { DivBy0Error.invalidNel } + def errorOrDiv(v1: Double, v2: Int): ErrorOr[Double] = if (v2 != 0) { Valid(v1.toDouble / v2.toDouble) } + else { DivBy0Error.invalidNel } + def errorOrSelect(v1: Int, + v2: Int, + v3: Int, + v4: Int, + v5: Int, + v6: Int, + v7: Int, + v8: Int, + v9: Int, + v10: Int, + v11: Int, + v12: Int, + v13: Int, + v14: Int, + v15: Int, + v16: Int, + v17: Int, + v18: Int, + v19: Int, + v20: Int, + v21: Int, + v22: Int + ): ErrorOr[Int] = Valid(v4 + v6 + v22) val valid0 = Valid(0) val valid1 = Valid(1) @@ -63,13 +84,29 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "flatMapN 22-tuples into a Valid" in { - (valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, - valid0, valid1, valid2, valid0) flatMapN errorOrSelect should be(Valid(0 + 2 + 0)) + (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ) flatMapN errorOrSelect should be(Valid(0 + 2 + 0)) } it should "flatMapN 1-tuples into a Valid string" in { @@ -139,82 +176,211 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { } it should "flatMapN 13-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0) - .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) + val result = + (valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0) + .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120")) } it should "flatMapN 14-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1) - .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) + val result = + (valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1) + .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("01201201201201")) } it should "flatMapN 15-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("012012012012012")) } it should "flatMapN 16-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120120")) } it should "flatMapN 17-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("01201201201201201")) } it should "flatMapN 18-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("012012012012012012")) } it should "flatMapN 19-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120120120")) } it should "flatMapN 20-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("01201201201201201201")) } it should "flatMapN 21-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2 + ) .flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("012012012012012012012")) } it should "flatMapN 22-tuples into a Valid string" in { - val result = ( - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, - valid0, valid1, valid2, valid0, valid1, valid2, valid0, valid1, valid2, valid0).flatMapN( - Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) + val result = (valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0, + valid1, + valid2, + valid0 + ).flatMapN(Array(_, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _, _).mkString.valid) result should be(Valid("0120120120120120120120")) } @@ -228,7 +394,8 @@ class ErrorOrSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { | def flatMapN[T_OUT](f1: (A) => ErrorOr[T_OUT]): ErrorOr[T_OUT] = t1.tupled flatMap f1.tupled |} | - |""".stripMargin) + |""".stripMargin + ) result } diff --git a/common/src/test/scala/common/validation/ValidationSpec.scala b/common/src/test/scala/common/validation/ValidationSpec.scala index 543e33d7573..8a1c72a922e 100644 --- a/common/src/test/scala/common/validation/ValidationSpec.scala +++ b/common/src/test/scala/common/validation/ValidationSpec.scala @@ -14,7 +14,6 @@ import common.mock.MockSugar import scala.util.{Failure, Success} - class ValidationSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with MockSugar { behavior of "Validation" diff --git a/core/src/main/resources/reference.conf b/core/src/main/resources/reference.conf index c58eda5bc9f..d5aff092bc3 100644 --- a/core/src/main/resources/reference.conf +++ b/core/src/main/resources/reference.conf @@ -15,6 +15,15 @@ webservice { } akka { + + http { + client { + parsing { + illegal-header-warnings = off + } + } + } + actor.default-dispatcher.fork-join-executor { # Number of threads = min(parallelism-factor * cpus, parallelism-max) # Below are the default values set by Akka, uncomment to tune these @@ -109,7 +118,8 @@ system { max-concurrent-workflows = 5000 # Cromwell will launch up to N submitted workflows at a time, regardless of how many open workflow slots exist - max-workflow-launch-count = 50 + # Deviating from 1 is not recommended for multi-runner setups due to possible deadlocks. [BW-962] + max-workflow-launch-count = 1 # Workflows will be grouped by the value of the specified field in their workflow options. # @@ -328,8 +338,8 @@ filesystems { # Class to instantiate and propagate to all factories. Takes a single typesafe config argument class = "cromwell.filesystems.drs.DrsFileSystemConfig" config { - martha { - url = "https://martha-url-here" + resolver { + url = "https://martha-url-here or https://drshub-url-here" # When true (and when possible) any DrsPath will be internally converted to another compatible Path such as a # GcsPath. This enables other optimizations such as using batch requests for GCS for hashes, using the # existing GCS downloader in Papi, etc. @@ -402,6 +412,15 @@ docker { max-retries = 3 // Supported registries (Docker Hub, Google, Quay) can have additional configuration set separately + azure { + // Worst case `ReadOps per minute` value from official docs + // https://github.com/MicrosoftDocs/azure-docs/blob/main/includes/container-registry-limits.md + throttle { + number-of-requests = 1000 + per = 60 seconds + } + num-threads = 10 + } google { // Example of how to configure throttling, available for all supported registries throttle { @@ -414,6 +433,8 @@ docker { } dockerhub.num-threads = 10 quay.num-threads = 10 + ecr.num-threads = 10 + ecr-public.num-threads = 10 } } @@ -439,6 +460,10 @@ engine { languages { default: WDL WDL { + http-allow-list { + enabled: false + allowed-http-hosts: [] + } versions { default: "draft-2" "draft-2" { @@ -457,34 +482,19 @@ languages { } } "biscayne" { - # WDL biscayne is our in-progress name for what will (probably) become WDL 1.1 + # WDL biscayne is our in-progress name for what will become WDL 1.1 language-factory = "languages.wdl.biscayne.WdlBiscayneLanguageFactory" config { strict-validation: true enabled: true } } - } - } - CWL { - versions { - default: "v1.0" - "v1.0" { - language-factory = "languages.cwl.CwlV1_0LanguageFactory" + "cascades" { + # WDL cascades is our in-progress name for what will (probably) become WDL 2.0 + language-factory = "languages.wdl.cascades.WdlCascadesLanguageFactory" config { - strict-validation: false + strict-validation: true enabled: true - - # Optional section to define a command returning paths to output files that - # can only be known after the user's command has been run - output-runtime-extractor { - # Optional docker image on which the command should be run - Must have bash if provided - docker-image = "stedolan/jq@sha256:a61ed0bca213081b64be94c5e1b402ea58bc549f457c2682a86704dd55231e09" - # Single line command executed after the tool has run. - # It should write to stdout the file paths that are referenced in the cwl.output.json (or any other file path - # not already defined in the outputs of the tool and therefore unknown when the tool is run) - command = "cat cwl.output.json 2>/dev/null | jq -r '.. | .path? // .location? // empty'" - } } } } @@ -516,6 +526,8 @@ backend { } } +# Note: When adding a new actor that uses service registry pattern make sure that the new actor handles the graceful +# shutdown command ('ShutdownCommand'). See https://github.com/broadinstitute/cromwell/issues/2575 services { KeyValue { class = "cromwell.services.keyvalue.impl.SqlKeyValueServiceActor" @@ -563,6 +575,15 @@ services { # Default - run within the Cromwell JVM class = "cromwell.services.womtool.impl.WomtoolServiceInCromwellActor" } + GithubAuthVending { + class = "cromwell.services.auth.impl.GithubAuthVendingActor" + config { + enabled = false + auth.azure = false + # Set this to the service that Cromwell should retrieve Github access token associated with user's token. + # ecm.base-url = "" + } + } } include required(classpath("reference_database.inc.conf")) @@ -619,3 +640,19 @@ ga4gh { contact-info-url = "https://cromwell.readthedocs.io/en/stable/" } } + +workflow-state-callback { + enabled: false + ## The number of threads to allocate for performing callbacks + # num-threads: 5 + # endpoint: "http://example.com/foo" # Can be overridden in workflow options + # auth.azure: true + ## Users can override default retry behavior if desired + # request-backoff { + # min: "3 seconds" + # max: "5 minutes" + # multiplier: 1.1 + # } + # max-retries = 10 + +} diff --git a/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala index 3000b30e8df..8c53b38c36d 100644 --- a/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala +++ b/core/src/main/scala/cromwell/core/BackendDockerConfiguration.scala @@ -14,7 +14,9 @@ object DockerCredentials { object DockerCredentialUsernameAndPassword { private val tokenStringFormat = raw"([^:]*):(.*)".r - def unapply(arg: DockerCredentials): Option[(String, String)] = Try(new String(Base64.getDecoder.decode(arg.token))).toOption match { + def unapply(arg: DockerCredentials): Option[(String, String)] = Try( + new String(Base64.getDecoder.decode(arg.token)) + ).toOption match { case Some(tokenStringFormat(username, password)) => Some((username, password)) case _ => None } diff --git a/core/src/main/scala/cromwell/core/ConfigUtil.scala b/core/src/main/scala/cromwell/core/ConfigUtil.scala index 0fd5002ffa8..5adf56ec0a4 100644 --- a/core/src/main/scala/cromwell/core/ConfigUtil.scala +++ b/core/src/main/scala/cromwell/core/ConfigUtil.scala @@ -8,7 +8,7 @@ import com.typesafe.config.{Config, ConfigException, ConfigValue} import org.slf4j.LoggerFactory import scala.jdk.CollectionConverters._ -import scala.reflect.{ClassTag, classTag} +import scala.reflect.{classTag, ClassTag} object ConfigUtil { @@ -20,12 +20,12 @@ object ConfigUtil { /** * For keys that are in the configuration but not in the reference keySet, log a warning. */ - def warnNotRecognized(keySet: Set[String], context: String) = { + def warnNotRecognized(keySet: Set[String], context: String) = keys.diff(keySet) match { - case warnings if warnings.nonEmpty => validationLogger.warn(s"Unrecognized configuration key(s) for $context: ${warnings.mkString(", ")}") + case warnings if warnings.nonEmpty => + validationLogger.warn(s"Unrecognized configuration key(s) for $context: ${warnings.mkString(", ")}") case _ => } - } /** * Validates that the value for this key is a well formed URL. @@ -34,15 +34,15 @@ object ConfigUtil { new URL(config.getString(url)) } - def validateString(key: String): ValidatedNel[String, String] = try { + def validateString(key: String): ValidatedNel[String, String] = try config.getString(key).validNel - } catch { + catch { case _: ConfigException.Missing => s"Could not find key: $key".invalidNel } - def validateConfig(key: String): ValidatedNel[String, Config] = try { + def validateConfig(key: String): ValidatedNel[String, Config] = try config.getConfig(key).validNel - } catch { + catch { case _: ConfigException.Missing => s"Could not find key: $key".invalidNel case _: ConfigException.WrongType => s"key $key cannot be parsed to a Config".invalidNel } @@ -50,6 +50,7 @@ object ConfigUtil { } implicit class EnhancedValidation[I <: AnyRef](val value: I) extends AnyVal { + /** * Validates this value by applying validationFunction to it and returning a Validation: * Returns successNel upon success. @@ -58,9 +59,9 @@ object ConfigUtil { * @tparam O return type of validationFunction * @tparam E Restricts the subtype of Exception that should be caught during validation */ - def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidatedNel[String, O] = try { + def validateAny[O, E <: Exception: ClassTag](validationFunction: I => O): ValidatedNel[String, O] = try validationFunction(value).validNel - } catch { + catch { case e if classTag[E].runtimeClass.isInstance(e) => e.getMessage.invalidNel } } diff --git a/core/src/main/scala/cromwell/core/DockerConfiguration.scala b/core/src/main/scala/cromwell/core/DockerConfiguration.scala index e765eec80fe..3f2dfa5c6f8 100644 --- a/core/src/main/scala/cromwell/core/DockerConfiguration.scala +++ b/core/src/main/scala/cromwell/core/DockerConfiguration.scala @@ -18,23 +18,30 @@ object DockerConfiguration { lazy val instance: DockerConfiguration = { if (dockerHashLookupConfig.hasPath("gcr-api-queries-per-100-seconds")) { - logger.warn("'docker.hash-lookup.gcr-api-queries-per-100-seconds' is no longer supported, use 'docker.hash-lookup.google.throttle' instead (see reference.conf)") + logger.warn( + "'docker.hash-lookup.gcr-api-queries-per-100-seconds' is no longer supported, use 'docker.hash-lookup.google.throttle' instead (see reference.conf)" + ) } - val enabled = validate { dockerHashLookupConfig.as[Boolean]("enabled") } - val cacheEntryTtl = validate { dockerHashLookupConfig.as[FiniteDuration]("cache-entry-ttl") } - val cacheSize = validate { dockerHashLookupConfig.as[Long]("cache-size") } - val method: ErrorOr[DockerHashLookupMethod] = validate { dockerHashLookupConfig.as[String]("method") } map { + val enabled = validate(dockerHashLookupConfig.as[Boolean]("enabled")) + val cacheEntryTtl = validate(dockerHashLookupConfig.as[FiniteDuration]("cache-entry-ttl")) + val cacheSize = validate(dockerHashLookupConfig.as[Long]("cache-size")) + val method: ErrorOr[DockerHashLookupMethod] = validate(dockerHashLookupConfig.as[String]("method")) map { case "local" => DockerLocalLookup case "remote" => DockerRemoteLookup case other => throw new IllegalArgumentException(s"Unrecognized docker hash lookup method: $other") } - val sizeCompressionFactor = validate { dockerHashLookupConfig.as[Double]("size-compression-factor") } - val maxTimeBetweenRetries = validate { dockerHashLookupConfig.as[FiniteDuration]("max-time-between-retries") } - val maxRetries = validate { dockerHashLookupConfig.as[Int]("max-retries") } + val sizeCompressionFactor = validate(dockerHashLookupConfig.as[Double]("size-compression-factor")) + val maxTimeBetweenRetries = validate(dockerHashLookupConfig.as[FiniteDuration]("max-time-between-retries")) + val maxRetries = validate(dockerHashLookupConfig.as[Int]("max-retries")) val dockerConfiguration = (enabled, - cacheEntryTtl, cacheSize, method, - sizeCompressionFactor, maxTimeBetweenRetries, maxRetries) mapN DockerConfiguration.apply + cacheEntryTtl, + cacheSize, + method, + sizeCompressionFactor, + maxTimeBetweenRetries, + maxRetries + ) mapN DockerConfiguration.apply dockerConfiguration match { case Valid(conf) => conf @@ -44,14 +51,14 @@ object DockerConfiguration { } case class DockerConfiguration( - enabled: Boolean, - cacheEntryTtl: FiniteDuration, - cacheSize: Long, - method: DockerHashLookupMethod, - sizeCompressionFactor: Double, - maxTimeBetweenRetries: FiniteDuration, - maxRetries: Int - ) + enabled: Boolean, + cacheEntryTtl: FiniteDuration, + cacheSize: Long, + method: DockerHashLookupMethod, + sizeCompressionFactor: Double, + maxTimeBetweenRetries: FiniteDuration, + maxRetries: Int +) sealed trait DockerHashLookupMethod diff --git a/core/src/main/scala/cromwell/core/Encryption.scala b/core/src/main/scala/cromwell/core/Encryption.scala index 0258d4511e3..4e05693b003 100644 --- a/core/src/main/scala/cromwell/core/Encryption.scala +++ b/core/src/main/scala/cromwell/core/Encryption.scala @@ -31,15 +31,18 @@ case object Aes256Cbc { cipher } - final def validateLength(arrayName: String, array: Array[Byte], expectedBitLength: Int): Try[Unit] = { + final def validateLength(arrayName: String, array: Array[Byte], expectedBitLength: Int): Try[Unit] = if (array.length * 8 == expectedBitLength) { Success(()) } else { - Failure(new IllegalArgumentException(s"$arrayName size (${array.length * 8} bits) did not match the required length $expectedBitLength")) + Failure( + new IllegalArgumentException( + s"$arrayName size (${array.length * 8} bits) did not match the required length $expectedBitLength" + ) + ) } - } - final def encrypt(plainText: Array[Byte], secretKey: SecretKey): Try[EncryptedBytes] = { + final def encrypt(plainText: Array[Byte], secretKey: SecretKey): Try[EncryptedBytes] = validateLength("Secret key", secretKey.key, keySize) map { _ => val iv = new Array[Byte](blockSize / 8) ranGen.nextBytes(iv) @@ -47,15 +50,15 @@ case object Aes256Cbc { val cipher = init(Cipher.ENCRYPT_MODE, secretKey.key, iv) EncryptedBytes(cipher.doFinal(plainText), iv) } - } - final def decrypt(encryptedBytes: EncryptedBytes, secretKey: SecretKey): Try[Array[Byte]] = { + final def decrypt(encryptedBytes: EncryptedBytes, secretKey: SecretKey): Try[Array[Byte]] = for { _ <- validateLength("Secret key", secretKey.key, keySize) _ <- validateLength("Initialization vector", encryptedBytes.initializationVector, blockSize) - bytes = init(Cipher.DECRYPT_MODE, secretKey.key, encryptedBytes.initializationVector).doFinal(encryptedBytes.cipherText) + bytes = init(Cipher.DECRYPT_MODE, secretKey.key, encryptedBytes.initializationVector).doFinal( + encryptedBytes.cipherText + ) } yield bytes - } } final case class EncryptedBytes(cipherText: Array[Byte], initializationVector: Array[Byte]) { @@ -71,4 +74,4 @@ object EncryptedBytes { final case class SecretKey(key: Array[Byte]) object SecretKey { def apply(base64KeyString: String): SecretKey = SecretKey(Base64.decodeBase64(base64KeyString)) -} \ No newline at end of file +} diff --git a/core/src/main/scala/cromwell/core/ExecutionIndex.scala b/core/src/main/scala/cromwell/core/ExecutionIndex.scala index 4f04179db0c..926fad13c11 100644 --- a/core/src/main/scala/cromwell/core/ExecutionIndex.scala +++ b/core/src/main/scala/cromwell/core/ExecutionIndex.scala @@ -23,8 +23,7 @@ object ExecutionIndex { } implicit val ExecutionIndexOrdering = new Ordering[ExecutionIndex] { - override def compare(x: ExecutionIndex, y: ExecutionIndex): Int = { + override def compare(x: ExecutionIndex, y: ExecutionIndex): Int = x.fromIndex.compareTo(y.fromIndex) - } } } diff --git a/core/src/main/scala/cromwell/core/ExecutionStatus.scala b/core/src/main/scala/cromwell/core/ExecutionStatus.scala index 3d4016d90a5..fa2ece67ab5 100644 --- a/core/src/main/scala/cromwell/core/ExecutionStatus.scala +++ b/core/src/main/scala/cromwell/core/ExecutionStatus.scala @@ -2,7 +2,8 @@ package cromwell.core object ExecutionStatus extends Enumeration { type ExecutionStatus = Value - val NotStarted, WaitingForQueueSpace, QueuedInCromwell, Starting, Running, Aborting, Failed, RetryableFailure, Done, Bypassed, Aborted, Unstartable = Value + val NotStarted, WaitingForQueueSpace, QueuedInCromwell, Starting, Running, Aborting, Failed, RetryableFailure, Done, + Bypassed, Aborted, Unstartable = Value val TerminalStatuses = Set(Failed, Done, Aborted, Bypassed, Unstartable) val TerminalOrRetryableStatuses = TerminalStatuses + RetryableFailure val NonTerminalStatuses = values.diff(TerminalOrRetryableStatuses) @@ -24,7 +25,7 @@ object ExecutionStatus extends Enumeration { case Done => 11 } } - + implicit class EnhancedExecutionStatus(val status: ExecutionStatus) extends AnyVal { def isTerminal: Boolean = TerminalStatuses contains status diff --git a/core/src/main/scala/cromwell/core/HogGroup.scala b/core/src/main/scala/cromwell/core/HogGroup.scala index 87bcad5d0c9..c49b84a9ecc 100644 --- a/core/src/main/scala/cromwell/core/HogGroup.scala +++ b/core/src/main/scala/cromwell/core/HogGroup.scala @@ -17,16 +17,16 @@ object HogGroup { if (config.hasPath("system.hog-safety.workflow-option")) { val hogGroupField = config.getString("system.hog-safety.workflow-option") - (options, workflowId) => { + (options, workflowId) => options.get(hogGroupField) match { case Success(hg) => HogGroup(hg) case Failure(_) => HogGroup(workflowId.shortString) } - } - } else { - (_, workflowId) => HogGroup(workflowId.shortString) + } else { (_, workflowId) => + HogGroup(workflowId.shortString) } } - def decide(workflowOptions: WorkflowOptions, workflowId: WorkflowId): HogGroup = HogGroupDeciderFunction.apply(workflowOptions, workflowId) + def decide(workflowOptions: WorkflowOptions, workflowId: WorkflowId): HogGroup = + HogGroupDeciderFunction.apply(workflowOptions, workflowId) } diff --git a/core/src/main/scala/cromwell/core/JobKey.scala b/core/src/main/scala/cromwell/core/JobKey.scala index e5f990aa433..99118dc2235 100644 --- a/core/src/main/scala/cromwell/core/JobKey.scala +++ b/core/src/main/scala/cromwell/core/JobKey.scala @@ -13,6 +13,6 @@ trait JobKey { import ExecutionIndex.IndexEnhancedIndex s"${getClass.getSimpleName}_${node.getClass.getSimpleName}_${node.fullyQualifiedName}:${index.fromIndex}:$attempt" } - - def isShard = index.isDefined + + def isShard = index.isDefined } diff --git a/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala b/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala index 53f0e4de6fc..8932b3f6836 100644 --- a/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala +++ b/core/src/main/scala/cromwell/core/MonitoringCompanionActor.scala @@ -9,21 +9,21 @@ import scala.language.postfixOps object MonitoringCompanionActor { sealed trait MonitoringCompanionCommand - private [core] case object AddWork extends MonitoringCompanionCommand - private [core] case object RemoveWork extends MonitoringCompanionCommand - private [core] def props(actorToMonitor: ActorRef) = Props(new MonitoringCompanionActor(actorToMonitor)) + private[core] case object AddWork extends MonitoringCompanionCommand + private[core] case object RemoveWork extends MonitoringCompanionCommand + private[core] def props(actorToMonitor: ActorRef) = Props(new MonitoringCompanionActor(actorToMonitor)) } -private [core] class MonitoringCompanionActor(actorToMonitor: ActorRef) extends Actor with ActorLogging { +private[core] class MonitoringCompanionActor(actorToMonitor: ActorRef) extends Actor with ActorLogging { private var workCount: Int = 0 - + override def receive = { case AddWork => workCount += 1 case RemoveWork => workCount -= 1 case ShutdownCommand if workCount <= 0 => context stop actorToMonitor context stop self - case ShutdownCommand => + case ShutdownCommand => log.info(s"{} is still processing {} messages", actorToMonitor.path.name, workCount) context.system.scheduler.scheduleOnce(1 second, self, ShutdownCommand)(context.dispatcher) () @@ -33,12 +33,12 @@ private [core] class MonitoringCompanionActor(actorToMonitor: ActorRef) extends trait MonitoringCompanionHelper { this: Actor => private val monitoringActor = context.actorOf(MonitoringCompanionActor.props(self)) private var shuttingDown: Boolean = false - + def addWork() = monitoringActor ! AddWork def removeWork() = monitoringActor ! RemoveWork val monitoringReceive: Receive = { - case ShutdownCommand if !shuttingDown => + case ShutdownCommand if !shuttingDown => shuttingDown = true monitoringActor ! ShutdownCommand case ShutdownCommand => // Ignore if we're already shutting down diff --git a/core/src/main/scala/cromwell/core/WorkflowId.scala b/core/src/main/scala/cromwell/core/WorkflowId.scala index feb0ee601a9..f2444738fed 100644 --- a/core/src/main/scala/cromwell/core/WorkflowId.scala +++ b/core/src/main/scala/cromwell/core/WorkflowId.scala @@ -8,19 +8,17 @@ sealed trait WorkflowId { override def toString = id.toString def shortString = id.toString.split("-")(0) - def toRoot: RootWorkflowId = { + def toRoot: RootWorkflowId = this match { case root: RootWorkflowId => root case _ => RootWorkflowId(id) } - } - def toPossiblyNotRoot: PossiblyNotRootWorkflowId = { + def toPossiblyNotRoot: PossiblyNotRootWorkflowId = this match { case possiblyNotRoot: PossiblyNotRootWorkflowId => possiblyNotRoot case _ => PossiblyNotRootWorkflowId(id) } - } } object WorkflowId { diff --git a/core/src/main/scala/cromwell/core/WorkflowOptions.scala b/core/src/main/scala/cromwell/core/WorkflowOptions.scala index 010300b2d8b..cbdb1201986 100644 --- a/core/src/main/scala/cromwell/core/WorkflowOptions.scala +++ b/core/src/main/scala/cromwell/core/WorkflowOptions.scala @@ -55,13 +55,14 @@ object WorkflowOptions { case object FinalWorkflowLogDir extends WorkflowOption("final_workflow_log_dir") case object FinalCallLogsDir extends WorkflowOption("final_call_logs_dir") case object FinalWorkflowOutputsDir extends WorkflowOption("final_workflow_outputs_dir") - case object UseRelativeOutputPaths extends WorkflowOption(name="use_relative_output_paths") + case object UseRelativeOutputPaths extends WorkflowOption(name = "use_relative_output_paths") // Misc. case object DefaultRuntimeOptions extends WorkflowOption("default_runtime_attributes") case object WorkflowFailureMode extends WorkflowOption("workflow_failure_mode") case object UseReferenceDisks extends WorkflowOption("use_reference_disks") case object MemoryRetryMultiplier extends WorkflowOption("memory_retry_multiplier") + case object WorkflowCallbackUri extends WorkflowOption("workflow_callback_uri") private lazy val WorkflowOptionsConf = ConfigFactory.load.getConfig("workflow-options") private lazy val EncryptedFields: Seq[String] = WorkflowOptionsConf.getStringList("encrypted-fields").asScala.toList @@ -69,23 +70,28 @@ object WorkflowOptions { private lazy val defaultRuntimeOptionKey: String = DefaultRuntimeOptions.name private lazy val validObjectKeys: Set[String] = Set(DefaultRuntimeOptions.name, "google_labels") - def encryptField(value: JsString): Try[JsObject] = { + def encryptField(value: JsString): Try[JsObject] = Aes256Cbc.encrypt(value.value.getBytes("utf-8"), SecretKey(EncryptionKey)) match { - case Success(encryptedValue) => Success(JsObject(Map( - "iv" -> JsString(encryptedValue.base64Iv), - "ciphertext" -> JsString(encryptedValue.base64CipherText) - ))) + case Success(encryptedValue) => + Success( + JsObject( + Map( + "iv" -> JsString(encryptedValue.base64Iv), + "ciphertext" -> JsString(encryptedValue.base64CipherText) + ) + ) + ) case Failure(ex) => Failure(ex) } - } - def decryptField(obj: JsObject): Try[String] = { + def decryptField(obj: JsObject): Try[String] = (obj.fields.get("iv"), obj.fields.get("ciphertext")) match { case (Some(iv: JsString), Some(ciphertext: JsString)) => - Aes256Cbc.decrypt(EncryptedBytes(ciphertext.value, iv.value), SecretKey(WorkflowOptions.EncryptionKey)).map(new String(_, "utf-8")) + Aes256Cbc + .decrypt(EncryptedBytes(ciphertext.value, iv.value), SecretKey(WorkflowOptions.EncryptionKey)) + .map(new String(_, "utf-8")) case _ => Failure(new RuntimeException(s"JsObject must have 'iv' and 'ciphertext' fields to decrypt: $obj")) } - } def isEncryptedField(jsValue: JsValue): Boolean = jsValue match { case obj: JsObject if obj.fields.keys.exists(_ == "iv") && obj.fields.keys.exists(_ == "ciphertext") => true @@ -101,7 +107,8 @@ object WorkflowOptions { case (k, v: JsNumber) => k -> Success(v) case (k, v) if isEncryptedField(v) => k -> Success(v) case (k, v: JsArray) => k -> Success(v) - case (k, v) => k -> Failure(new UnsupportedOperationException(s"Unsupported key/value pair in WorkflowOptions: $k -> $v")) + case (k, v) => + k -> Failure(new UnsupportedOperationException(s"Unsupported key/value pair in WorkflowOptions: $k -> $v")) } encrypted.values collect { case f: Failure[_] => f } match { @@ -116,7 +123,7 @@ object WorkflowOptions { case Success(x) => Failure(new UnsupportedOperationException(s"Expecting JSON object, got $x")) } - def fromMap(m: Map[String, String]) = fromJsonObject(JsObject(m map { case (k, v) => k -> JsString(v)})) + def fromMap(m: Map[String, String]) = fromJsonObject(JsObject(m map { case (k, v) => k -> JsString(v) })) private def getAsJson(key: String, jsObject: JsObject) = jsObject.fields.get(key) match { case Some(jsStr: JsString) => Success(jsStr) @@ -156,7 +163,7 @@ case class WorkflowOptions(jsObject: JsObject) { } def getVectorOfStrings(key: String): ErrorOr[Option[Vector[String]]] = jsObject.fields.get(key) match { - case Some(jsArr: JsArray) => Option(jsArr.elements collect { case e: JsString => e.value } ).validNel + case Some(jsArr: JsArray) => Option(jsArr.elements collect { case e: JsString => e.value }).validNel case Some(jsVal: JsValue) => s"Unsupported JsValue as JsArray: $jsVal".invalidNel case _ => None.validNel } @@ -166,8 +173,14 @@ case class WorkflowOptions(jsObject: JsObject) { } lazy val defaultRuntimeOptions = jsObject.fields.get(defaultRuntimeOptionKey) match { - case Some(jsObj: JsObject) => TryUtil.sequenceMap(jsObj.fields map { case (k, _) => k -> WorkflowOptions.getAsJson(k, jsObj) }) - case Some(jsVal) => Failure(new IllegalArgumentException(s"Unsupported JsValue for $defaultRuntimeOptionKey: $jsVal. Expected a JSON object.")) + case Some(jsObj: JsObject) => + TryUtil.sequenceMap(jsObj.fields map { case (k, _) => k -> WorkflowOptions.getAsJson(k, jsObj) }) + case Some(jsVal) => + Failure( + new IllegalArgumentException( + s"Unsupported JsValue for $defaultRuntimeOptionKey: $jsVal. Expected a JSON object." + ) + ) case None => Failure(OptionNotFoundException(s"Cannot find definition for default runtime attributes")) } diff --git a/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala b/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala index ab80bf78724..e2db6df75c8 100644 --- a/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala +++ b/core/src/main/scala/cromwell/core/WorkflowProcessingEvents.scala @@ -36,4 +36,8 @@ object WorkflowProcessingEvents { val ProcessingEventsKey = "workflowProcessingEvents" } -case class WorkflowProcessingEvent(cromwellId: String, description: String, timestamp: OffsetDateTime, cromwellVersion: String) +case class WorkflowProcessingEvent(cromwellId: String, + description: String, + timestamp: OffsetDateTime, + cromwellVersion: String +) diff --git a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala index ee936b86819..aa1faf89542 100644 --- a/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala +++ b/core/src/main/scala/cromwell/core/WorkflowSourceFilesCollection.scala @@ -22,16 +22,15 @@ sealed trait WorkflowSourceFilesCollection { def importsZipFileOption: Option[Array[Byte]] = this match { case _: WorkflowSourceFilesWithoutImports => None - case w: WorkflowSourceFilesWithDependenciesZip => Option(w.importsZip) // i.e. Some(importsZip) if our wiring is correct + case w: WorkflowSourceFilesWithDependenciesZip => + Option(w.importsZip) // i.e. Some(importsZip) if our wiring is correct } - def setOptions(workflowOptions: WorkflowOptions) = { - + def setOptions(workflowOptions: WorkflowOptions) = this match { case w: WorkflowSourceFilesWithoutImports => w.copy(workflowOptions = workflowOptions) case w: WorkflowSourceFilesWithDependenciesZip => w.copy(workflowOptions = workflowOptions) } - } } trait HasWorkflowIdAndSources { @@ -51,7 +50,8 @@ object WorkflowSourceFilesCollection { importsFile: Option[Array[Byte]], workflowOnHold: Boolean, warnings: Seq[String], - requestedWorkflowId: Option[WorkflowId]): WorkflowSourceFilesCollection = importsFile match { + requestedWorkflowId: Option[WorkflowId] + ): WorkflowSourceFilesCollection = importsFile match { case Some(imports) => WorkflowSourceFilesWithDependenciesZip( workflowSource = workflowSource, @@ -65,7 +65,8 @@ object WorkflowSourceFilesCollection { importsZip = imports, workflowOnHold = workflowOnHold, warnings = warnings, - requestedWorkflowId = requestedWorkflowId) + requestedWorkflowId = requestedWorkflowId + ) case None => WorkflowSourceFilesWithoutImports( workflowSource = workflowSource, @@ -78,7 +79,8 @@ object WorkflowSourceFilesCollection { labelsJson = labelsJson, workflowOnHold = workflowOnHold, warnings = warnings, - requestedWorkflowId = requestedWorkflowId) + requestedWorkflowId = requestedWorkflowId + ) } } @@ -92,7 +94,8 @@ final case class WorkflowSourceFilesWithoutImports(workflowSource: Option[Workfl labelsJson: WorkflowJson, workflowOnHold: Boolean = false, warnings: Seq[String], - requestedWorkflowId: Option[WorkflowId]) extends WorkflowSourceFilesCollection + requestedWorkflowId: Option[WorkflowId] +) extends WorkflowSourceFilesCollection final case class WorkflowSourceFilesWithDependenciesZip(workflowSource: Option[WorkflowSource], workflowUrl: Option[WorkflowUrl], @@ -105,9 +108,9 @@ final case class WorkflowSourceFilesWithDependenciesZip(workflowSource: Option[W importsZip: Array[Byte], workflowOnHold: Boolean = false, warnings: Seq[String], - requestedWorkflowId: Option[WorkflowId]) extends WorkflowSourceFilesCollection { - override def toString = { + requestedWorkflowId: Option[WorkflowId] +) extends WorkflowSourceFilesCollection { + override def toString = s"WorkflowSourceFilesWithDependenciesZip($workflowSource, $workflowUrl, $workflowType, $workflowTypeVersion," + s""" $inputsJson, ${workflowOptions.asPrettyJson}, $labelsJson, <>, $warnings)""" - } } diff --git a/core/src/main/scala/cromwell/core/WorkflowState.scala b/core/src/main/scala/cromwell/core/WorkflowState.scala index db4bddfedc5..26294ff648b 100644 --- a/core/src/main/scala/cromwell/core/WorkflowState.scala +++ b/core/src/main/scala/cromwell/core/WorkflowState.scala @@ -2,7 +2,6 @@ package cromwell.core import cats.Semigroup - sealed trait WorkflowState { def isTerminal: Boolean protected def ordinal: Int @@ -10,10 +9,18 @@ sealed trait WorkflowState { } object WorkflowState { - lazy val WorkflowStateValues = Seq(WorkflowOnHold, WorkflowSubmitted, WorkflowRunning, WorkflowFailed, WorkflowSucceeded, WorkflowAborting, WorkflowAborted) + lazy val WorkflowStateValues = Seq(WorkflowOnHold, + WorkflowSubmitted, + WorkflowRunning, + WorkflowFailed, + WorkflowSucceeded, + WorkflowAborting, + WorkflowAborted + ) - def withName(str: String): WorkflowState = WorkflowStateValues.find(_.toString.equalsIgnoreCase(str)).getOrElse( - throw new NoSuchElementException(s"No such WorkflowState: $str")) + def withName(str: String): WorkflowState = WorkflowStateValues + .find(_.toString.equalsIgnoreCase(str)) + .getOrElse(throw new NoSuchElementException(s"No such WorkflowState: $str")) implicit val WorkflowStateSemigroup = new Semigroup[WorkflowState] { override def combine(f1: WorkflowState, f2: WorkflowState): WorkflowState = f1.combine(f2) @@ -22,7 +29,7 @@ object WorkflowState { implicit val WorkflowStateOrdering = Ordering.by { self: WorkflowState => self.ordinal } } -case object WorkflowOnHold extends WorkflowState{ +case object WorkflowOnHold extends WorkflowState { override val toString: String = "On Hold" override val isTerminal = false override val ordinal = 0 diff --git a/core/src/main/scala/cromwell/core/actor/BatchActor.scala b/core/src/main/scala/cromwell/core/actor/BatchActor.scala index 5988dd822d2..9b002f4fcf5 100644 --- a/core/src/main/scala/cromwell/core/actor/BatchActor.scala +++ b/core/src/main/scala/cromwell/core/actor/BatchActor.scala @@ -14,7 +14,6 @@ import scala.concurrent.Future import scala.concurrent.duration.{Duration, FiniteDuration} import scala.util.{Failure, Success} - /** A collection of state, data, and message types to support BatchActor. */ object BatchActor { type BatchData[C] = WeightedQueue[C, Int] @@ -45,8 +44,9 @@ object BatchActor { * It is backed by a WeightedQueue which makes it possible to decouple the number of messages received from * the effective "weight" of the queue. */ -abstract class BatchActor[C](val flushRate: FiniteDuration, - val batchSize: Int) extends FSM[BatchActorState, BatchData[C]] with Timers { +abstract class BatchActor[C](val flushRate: FiniteDuration, val batchSize: Int) + extends FSM[BatchActorState, BatchData[C]] + with Timers { private var shuttingDown: Boolean = false implicit val ec = context.dispatcher @@ -63,7 +63,8 @@ abstract class BatchActor[C](val flushRate: FiniteDuration, protected def routed: Boolean = false override def preStart(): Unit = { - if (logOnStartUp) log.info("{} configured to flush with batch size {} and process rate {}.", name, batchSize, flushRate) + if (logOnStartUp) + log.info("{} configured to flush with batch size {} and process rate {}.", name, batchSize, flushRate) if (flushRate != Duration.Zero) { timers.startPeriodicTimer(ScheduledFlushKey, ScheduledProcessAction, flushRate) } @@ -133,10 +134,9 @@ abstract class BatchActor[C](val flushRate: FiniteDuration, */ protected def process(data: NonEmptyVector[C]): Future[Int] - private def processIfBatchSizeReached(data: BatchData[C]) = { + private def processIfBatchSizeReached(data: BatchData[C]) = if (data.weight >= batchSize) processHead(data) else goto(WaitingToProcess) using data - } private def processHead(data: BatchData[C]) = if (data.innerQueue.nonEmpty) { val (head, newQueue) = data.behead(batchSize) @@ -159,8 +159,8 @@ abstract class BatchActor[C](val flushRate: FiniteDuration, goto(Processing) using newQueue } else - // This goto is important, even if we're already in WaitingToProcess we want to trigger the onTransition below - // to check if it's time to shutdown + // This goto is important, even if we're already in WaitingToProcess we want to trigger the onTransition below + // to check if it's time to shutdown goto(WaitingToProcess) using data onTransition { diff --git a/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala index d8b0cbe0716..85376bedb37 100644 --- a/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala +++ b/core/src/main/scala/cromwell/core/actor/RobustClientHelper.scala @@ -15,14 +15,14 @@ object RobustClientHelper { } trait RobustClientHelper { this: Actor with ActorLogging => - private [actor] implicit val robustActorHelperEc = context.dispatcher + implicit private[actor] val robustActorHelperEc = context.dispatcher private var backoff: Option[Backoff] = None // package private for testing - private [core] var timeouts = Map.empty[Any, (Cancellable, FiniteDuration)] + private[core] var timeouts = Map.empty[Any, (Cancellable, FiniteDuration)] - protected def initialBackoff(): Backoff = SimpleExponentialBackoff(5.seconds, 20.minutes, 2D) + protected def initialBackoff(): Backoff = SimpleExponentialBackoff(5.seconds, 20.minutes, 2d) def robustReceive: Receive = { case BackPressure(request) => @@ -33,39 +33,38 @@ trait RobustClientHelper { this: Actor with ActorLogging => case RequestTimeout(request, to) => onTimeout(request, to) } - private final def newTimer(msg: Any, to: ActorRef, in: FiniteDuration) = { + final private def newTimer(msg: Any, to: ActorRef, in: FiniteDuration) = context.system.scheduler.scheduleOnce(in, to, msg)(robustActorHelperEc, self) - } def robustSend(msg: Any, to: ActorRef, timeout: FiniteDuration = DefaultRequestLostTimeout): Unit = { to ! msg addTimeout(msg, to, timeout) } - private final def addTimeout(command: Any, to: ActorRef, timeout: FiniteDuration) = { + final private def addTimeout(command: Any, to: ActorRef, timeout: FiniteDuration) = { val cancellable = newTimer(RequestTimeout(command, to), self, timeout) timeouts = timeouts + (command -> (cancellable -> timeout)) } - protected final def hasTimeout(command: Any) = timeouts.get(command).isDefined + final protected def hasTimeout(command: Any) = timeouts.get(command).isDefined - protected final def cancelTimeout(command: Any) = { + final protected def cancelTimeout(command: Any) = { timeouts.get(command) foreach { case (cancellable, _) => cancellable.cancel() } timeouts = timeouts - command } - private final def resetTimeout(command: Any, to: ActorRef) = { + final private def resetTimeout(command: Any, to: ActorRef) = { val timeout = timeouts.get(command) map { _._2 } cancelTimeout(command) timeout foreach { addTimeout(command, to, _) } } - private [actor] final def generateBackpressureTime: FiniteDuration = { - val effectiveBackoff = backoff.getOrElse({ + final private[actor] def generateBackpressureTime: FiniteDuration = { + val effectiveBackoff = backoff.getOrElse { val firstBackoff = initialBackoff() backoff = Option(firstBackoff) firstBackoff - }) + } val backoffTime = effectiveBackoff.backoffMillis backoff = Option(effectiveBackoff.next) backoffTime.millis diff --git a/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala b/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala index d03853e5585..023a01dc450 100644 --- a/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala +++ b/core/src/main/scala/cromwell/core/actor/StreamActorHelper.scala @@ -13,8 +13,8 @@ import scala.concurrent.{ExecutionContext, Future} import scala.util.{Failure, Success} object StreamActorHelper { - private [actor] case class StreamFailed(failure: Throwable) - private [actor] case object StreamCompleted + private[actor] case class StreamFailed(failure: Throwable) + private[actor] case object StreamCompleted class ActorRestartException(throwable: Throwable) extends RuntimeException(throwable) } @@ -25,71 +25,68 @@ trait StreamActorHelper[T <: StreamContext] { this: Actor with ActorLogging => implicit def materializer: ActorMaterializer private val decider: Supervision.Decider = _ => Supervision.Resume - - private val replySink = Sink.foreach[(Any, T)] { - case (response, commandContext) => - val reply = commandContext.clientContext map { (_, response) } getOrElse response - commandContext.replyTo ! reply + + private val replySink = Sink.foreach[(Any, T)] { case (response, commandContext) => + val reply = commandContext.clientContext map { (_, response) } getOrElse response + commandContext.replyTo ! reply } protected def actorReceive: Receive - + protected def streamSource: Source[(Any, T), SourceQueueWithComplete[T]] override def receive = streamReceive.orElse(actorReceive) - + protected def onBackpressure(scale: Option[Double] = None): Unit = {} - private [actor] lazy val stream = { + private[actor] lazy val stream = streamSource .to(replySink) .withAttributes(ActorAttributes.supervisionStrategy(decider)) .run() - } - override def preStart(): Unit = { + override def preStart(): Unit = stream.watchCompletion() onComplete { case Success(_) => self ! StreamCompleted case Failure(failure) => self ! StreamFailed(failure) } - } def sendToStream(commandContext: T) = { val enqueue = stream offer commandContext map { case Enqueued => EnqueueResponse(Enqueued, commandContext) case other => EnqueueResponse(other, commandContext) - } recoverWith { - case t => Future.successful(FailedToEnqueue(t, commandContext)) + } recoverWith { case t => + Future.successful(FailedToEnqueue(t, commandContext)) } pipe(enqueue) to self () } - + private def backpressure(commandContext: StreamContext) = { - val originalRequest = commandContext.clientContext map { _ -> commandContext.request } getOrElse commandContext.request + val originalRequest = commandContext.clientContext map { + _ -> commandContext.request + } getOrElse commandContext.request commandContext.replyTo ! BackPressure(originalRequest) onBackpressure() } private def streamReceive: Receive = { - case ShutdownCommand => + case ShutdownCommand => stream.complete() case EnqueueResponse(Enqueued, _: T @unchecked) => // Good ! - case EnqueueResponse(_, commandContext) => backpressure(commandContext) case FailedToEnqueue(_, commandContext) => backpressure(commandContext) - - case StreamCompleted => + + case StreamCompleted => context stop self - case StreamFailed(failure) => + case StreamFailed(failure) => restart(failure) } /** Throw the exception to force the actor to restart so it can be back in business * IMPORTANT: Make sure the supervision strategy for this actor is Restart */ - private def restart(throwable: Throwable) = { + private def restart(throwable: Throwable) = throw new ActorRestartException(throwable) - } } diff --git a/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala b/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala index c1128897445..6ddbb5f8b18 100644 --- a/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala +++ b/core/src/main/scala/cromwell/core/actor/ThrottlerActor.scala @@ -15,7 +15,7 @@ import scala.concurrent.duration._ abstract class ThrottlerActor[C] extends BatchActor[C](Duration.Zero, 1) { override protected def logOnStartUp = false override def weightFunction(command: C) = 1 - override final def process(data: NonEmptyVector[C]): Future[Int] = { + final override def process(data: NonEmptyVector[C]): Future[Int] = // This ShouldNotBePossible™ but in case it happens, instead of dropping elements process them all anyway // Explanation: batch size is 1 which means as soon as we receive 1 element, the process method should be called. // Because the BatchActor calls the process method with vector of elements which total weight is batch size, and because @@ -25,6 +25,5 @@ abstract class ThrottlerActor[C] extends BatchActor[C](Duration.Zero, 1) { log.error("{} is throttled and is not supposed to process more than one element at a time !", self.path.name) data.toVector.traverse(processHead).map(_.length) } else processHead(data.head).map(_ => 1) - } def processHead(head: C): Future[Int] } diff --git a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala index 23d4f396a2c..08527dbcf1d 100644 --- a/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala +++ b/core/src/main/scala/cromwell/core/callcaching/CallCachingMode.scala @@ -1,6 +1,7 @@ package cromwell.core.callcaching sealed trait CallCachingMode { + /** * Return an equivalent of this call caching mode with READ disabled. */ @@ -19,11 +20,14 @@ case object CallCachingOff extends CallCachingMode { override val withoutWrite = this } -case class CallCachingActivity(readWriteMode: ReadWriteMode, options: CallCachingOptions = CallCachingOptions()) extends CallCachingMode { +case class CallCachingActivity(readWriteMode: ReadWriteMode, options: CallCachingOptions = CallCachingOptions()) + extends CallCachingMode { override val readFromCache = readWriteMode.r override val writeToCache = readWriteMode.w - override lazy val withoutRead: CallCachingMode = if (!writeToCache) CallCachingOff else this.copy(readWriteMode = WriteCache) - override lazy val withoutWrite: CallCachingMode = if (!readFromCache) CallCachingOff else this.copy(readWriteMode = ReadCache) + override lazy val withoutRead: CallCachingMode = + if (!writeToCache) CallCachingOff else this.copy(readWriteMode = WriteCache) + override lazy val withoutWrite: CallCachingMode = + if (!readFromCache) CallCachingOff else this.copy(readWriteMode = ReadCache) override val toString = readWriteMode.toString } @@ -35,4 +39,6 @@ case object ReadCache extends ReadWriteMode { override val w = false } case object WriteCache extends ReadWriteMode { override val r = false } case object ReadAndWriteCache extends ReadWriteMode -final case class CallCachingOptions(invalidateBadCacheResults: Boolean = true, workflowOptionCallCachePrefixes: Option[Vector[String]] = None) +final case class CallCachingOptions(invalidateBadCacheResults: Boolean = true, + workflowOptionCallCachePrefixes: Option[Vector[String]] = None +) diff --git a/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala b/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala index 920702280b0..a78c7f20b6a 100644 --- a/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala +++ b/core/src/main/scala/cromwell/core/callcaching/HashResultMessage.scala @@ -2,7 +2,6 @@ package cromwell.core.callcaching import cromwell.core.callcaching.HashKey.KeySeparator - object HashKey { private val KeySeparator = ": " def apply(keyComponents: String*) = new HashKey(true, keyComponents.toList) diff --git a/core/src/main/scala/cromwell/core/core.scala b/core/src/main/scala/cromwell/core/core.scala index 68edbaacbe7..2535a58f5c6 100644 --- a/core/src/main/scala/cromwell/core/core.scala +++ b/core/src/main/scala/cromwell/core/core.scala @@ -7,7 +7,6 @@ import mouse.boolean._ import scala.concurrent.duration.FiniteDuration - case class StandardPaths(output: Path, error: Path) case class CallContext(root: Path, standardPaths: StandardPaths, isDocker: Boolean) @@ -27,26 +26,35 @@ object CromwellFatalException { class CromwellFatalException(val exception: Throwable) extends Exception(exception) with CromwellFatalExceptionMarker case class CromwellAggregatedException(throwables: Seq[Throwable], exceptionContext: String = "") - extends Exception with ThrowableAggregation + extends Exception + with ThrowableAggregation case class CacheConfig(concurrency: Int, size: Long, ttl: FiniteDuration) import net.ceedubs.ficus.Ficus._ object CacheConfig { + /** * From an optional `Config` entry and specified defaults, always return a `CacheConfig` object. */ - def config(caching: Option[Config], defaultConcurrency: Int, defaultSize: Long, defaultTtl: FiniteDuration): CacheConfig = { + def config(caching: Option[Config], + defaultConcurrency: Int, + defaultSize: Long, + defaultTtl: FiniteDuration + ): CacheConfig = caching flatMap { c => optionalConfig(c, defaultConcurrency = defaultConcurrency, defaultSize = defaultSize, defaultTtl = defaultTtl) } getOrElse CacheConfig(concurrency = defaultConcurrency, size = defaultSize, ttl = defaultTtl) - } /** * From a non-optional `Config` and specified defaults, if caching is enabled return a `CacheConfig` object wrapped in a `Some`, * otherwise return `None`. */ - def optionalConfig(caching: Config, defaultConcurrency: Int, defaultSize: Long, defaultTtl: FiniteDuration): Option[CacheConfig] = { + def optionalConfig(caching: Config, + defaultConcurrency: Int, + defaultSize: Long, + defaultTtl: FiniteDuration + ): Option[CacheConfig] = { val cachingEnabled = caching.getOrElse("enabled", false) cachingEnabled.option( diff --git a/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala b/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala index caeb56509fd..ac2fe473572 100644 --- a/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala +++ b/core/src/main/scala/cromwell/core/filesystem/CromwellFileSystems.scala @@ -24,23 +24,29 @@ import scala.util.{Failure, Try} */ class CromwellFileSystems(globalConfig: Config) { // Validate the configuration and creates a Map of PathBuilderFactory constructors, along with their optional singleton config - private [filesystem] val factoryBuilders: Map[String, (Constructor[_], Option[AnyRef])] = if (globalConfig.hasPath("filesystems")) { - val rawConfigSet = globalConfig.getObject("filesystems").entrySet.asScala - val configMap = rawConfigSet.toList.map({ entry => entry.getKey -> entry.getValue }) - val constructorMap = configMap.traverse[ErrorOr, (String, (Constructor[_], Option[AnyRef]))]({ - case (key, fsConfig: ConfigObject) => processFileSystem(key, fsConfig) - case (key, _) => s"Invalid filesystem configuration for $key".invalidNel - }).map(_.toMap) - - constructorMap.unsafe("Failed to initialize Cromwell filesystems") - } else Map.empty + private[filesystem] val factoryBuilders: Map[String, (Constructor[_], Option[AnyRef])] = + if (globalConfig.hasPath("filesystems")) { + val rawConfigSet = globalConfig.getObject("filesystems").entrySet.asScala + val configMap = rawConfigSet.toList.map(entry => entry.getKey -> entry.getValue) + val constructorMap = configMap + .traverse[ErrorOr, (String, (Constructor[_], Option[AnyRef]))] { + case (key, fsConfig: ConfigObject) => processFileSystem(key, fsConfig) + case (key, _) => s"Invalid filesystem configuration for $key".invalidNel + } + .map(_.toMap) + + constructorMap.unsafe("Failed to initialize Cromwell filesystems") + } else Map.empty val supportedFileSystems: Iterable[String] = factoryBuilders.keys // Generate the appropriate constructor and optional singleton instance for a filesystem - private def processFileSystem(key: String, fsConfig: ConfigObject): ErrorOr[(String, (Constructor[_], Option[AnyRef]))] = { + private def processFileSystem(key: String, + fsConfig: ConfigObject + ): ErrorOr[(String, (Constructor[_], Option[AnyRef]))] = { // This is the (optional) singleton instance shared by all factory instances - val singletonInstance: Checked[Option[AnyRef]] = fsConfig.toConfig.getAs[Config]("global") + val singletonInstance: Checked[Option[AnyRef]] = fsConfig.toConfig + .getAs[Config]("global") .map(c => instantiateSingletonConfig(key, c).toValidated) .sequence[ErrorOr, AnyRef] .toEither @@ -57,70 +63,88 @@ class CromwellFileSystems(globalConfig: Config) { } // Instantiates the singleton config for a filesystem - private def instantiateSingletonConfig(filesystem: String, config: Config): Checked[AnyRef] = { + private def instantiateSingletonConfig(filesystem: String, config: Config): Checked[AnyRef] = for { constructor <- createConstructor(filesystem, config, List(classOf[Config])) instanceConfig = config.getAs[Config]("config").getOrElse(ConfigFactory.empty) instance <- Try(constructor.newInstance(instanceConfig)).toChecked - cast <- instance.cast[AnyRef].toChecked(s"The filesystem global configuration class for $filesystem is not a Java Object") + cast <- instance + .cast[AnyRef] + .toChecked(s"The filesystem global configuration class for $filesystem is not a Java Object") } yield cast - } // Create a constructor from a configuration object - private def createConstructor(key: String, config: Config, parameterTypes: List[Class[_]]): Checked[Constructor[_]] = for { - clazz <- config.as[Option[String]]("class").toChecked(s"Filesystem configuration $key doesn't have a class field") - constructor <- createConstructor(key, clazz, parameterTypes) - } yield constructor + private def createConstructor(key: String, config: Config, parameterTypes: List[Class[_]]): Checked[Constructor[_]] = + for { + clazz <- config.as[Option[String]]("class").toChecked(s"Filesystem configuration $key doesn't have a class field") + constructor <- createConstructor(key, clazz, parameterTypes) + } yield constructor // Getting a constructor from a class name - private def createConstructor(filesystem: String, className: String, parameterTypes: List[Class[_]]): Checked[Constructor[_]] = Try ( + private def createConstructor(filesystem: String, + className: String, + parameterTypes: List[Class[_]] + ): Checked[Constructor[_]] = Try( Class.forName(className).getConstructor(parameterTypes: _*) - ).recoverWith({ - case e: ClassNotFoundException => Failure( - new RuntimeException(s"Class $className for filesystem $filesystem cannot be found in the class path.", e) - ) - case e: NoSuchMethodException => Failure( - new RuntimeException(s"Class $className for filesystem $filesystem does not have the required constructor signature: (${parameterTypes.map(_.getCanonicalName).mkString(", ")})", e) - ) - }).toChecked + ).recoverWith { + case e: ClassNotFoundException => + Failure( + new RuntimeException(s"Class $className for filesystem $filesystem cannot be found in the class path.", e) + ) + case e: NoSuchMethodException => + Failure( + new RuntimeException( + s"Class $className for filesystem $filesystem does not have the required constructor signature: (${parameterTypes.map(_.getCanonicalName).mkString(", ")})", + e + ) + ) + }.toChecked // Instantiate a PathBuilderFactory from its constructor and instance config - private def instantiate(name: String, constructor: Constructor[_], instanceConfig: Config, global: Option[AnyRef]): Checked[PathBuilderFactory] = { + private def instantiate(name: String, + constructor: Constructor[_], + instanceConfig: Config, + global: Option[AnyRef] + ): Checked[PathBuilderFactory] = for { instance <- global match { case Some(g) => Try(constructor.newInstance(globalConfig, instanceConfig, g)).toChecked case None => Try(constructor.newInstance(globalConfig, instanceConfig)).toChecked } - cast <- instance.cast[PathBuilderFactory].toChecked(s"The filesystem class for $name is not an instance of PathBuilderFactory") + cast <- instance + .cast[PathBuilderFactory] + .toChecked(s"The filesystem class for $name is not an instance of PathBuilderFactory") } yield cast - } // Look for a constructor in the map of known filesystems private def getConstructor(fileSystemName: String): Checked[(Constructor[_], Option[AnyRef])] = factoryBuilders .get(fileSystemName) - .toChecked(s"Cannot find a filesystem with name $fileSystemName in the configuration. Available filesystems: ${factoryBuilders.keySet.mkString(", ")}") + .toChecked( + s"Cannot find a filesystem with name $fileSystemName in the configuration. Available filesystems: ${factoryBuilders.keySet + .mkString(", ")}" + ) /** * Try to find a configured filesystem with the given name and build a PathFactory for it * @param name name of the filesystem * @param instanceConfig filesystem specific configuration for this instance of the factory to build */ - def buildFactory(name: String, instanceConfig: Config): Checked[PathBuilderFactory] = { + def buildFactory(name: String, instanceConfig: Config): Checked[PathBuilderFactory] = if (DefaultPathBuilderFactory.name.equalsIgnoreCase(name)) DefaultPathBuilderFactory.validNelCheck - else for { - constructorAndGlobal <- getConstructor(name) - factory <- instantiate(name, constructorAndGlobal._1, instanceConfig, constructorAndGlobal._2) - } yield factory - } + else + for { + constructorAndGlobal <- getConstructor(name) + factory <- instantiate(name, constructorAndGlobal._1, instanceConfig, constructorAndGlobal._2) + } yield factory /** * Given a filesystems config, build the PathBuilderFactories */ - def factoriesFromConfig(filesystemsConfig: Config): Checked[Map[String, PathBuilderFactory]] = { + def factoriesFromConfig(filesystemsConfig: Config): Checked[Map[String, PathBuilderFactory]] = if (filesystemsConfig.hasPath("filesystems")) { // Iterate over the config entries under the "filesystems" config val rawConfigSet = filesystemsConfig.getObject("filesystems").entrySet().asScala - val configMap = rawConfigSet.toList.map({ entry => entry.getKey -> entry.getValue }) + val configMap = rawConfigSet.toList.map(entry => entry.getKey -> entry.getValue) import net.ceedubs.ficus.Ficus._ def isFilesystemEnabled(configObject: ConfigObject): Boolean = { @@ -141,7 +165,6 @@ class CromwellFileSystems(globalConfig: Config) { case (key, _) => s"Invalid filesystem backend configuration for $key".invalidNel } map { _.toMap } toEither } else Map.empty[String, PathBuilderFactory].validNelCheck - } } object CromwellFileSystems { diff --git a/core/src/main/scala/cromwell/core/io/AsyncIo.scala b/core/src/main/scala/cromwell/core/io/AsyncIo.scala index 435058d942c..535c7e1a8a4 100644 --- a/core/src/main/scala/cromwell/core/io/AsyncIo.scala +++ b/core/src/main/scala/cromwell/core/io/AsyncIo.scala @@ -23,7 +23,8 @@ object AsyncIo { */ class AsyncIo(ioEndpoint: ActorRef, ioCommandBuilder: IoCommandBuilder) { private def asyncCommand[A](commandTry: Try[IoCommand[A]], - timeout: FiniteDuration = AsyncIo.defaultTimeout): Future[A] = { + timeout: FiniteDuration = AsyncIo.defaultTimeout + ): Future[A] = commandTry match { case Failure(throwable) => Future.failed(throwable) @@ -32,46 +33,36 @@ class AsyncIo(ioEndpoint: ActorRef, ioCommandBuilder: IoCommandBuilder) { ioEndpoint ! commandWithPromise commandWithPromise.promise.future } - } /** * IMPORTANT: This loads the entire content of the file into memory ! * Only use for small files ! */ - def contentAsStringAsync(path: Path, maxBytes: Option[Int], failOnOverflow: Boolean): Future[String] = { + def contentAsStringAsync(path: Path, maxBytes: Option[Int], failOnOverflow: Boolean): Future[String] = asyncCommand(ioCommandBuilder.contentAsStringCommand(path, maxBytes, failOnOverflow)) - } - def writeAsync(path: Path, content: String, options: OpenOptions, compressPayload: Boolean = false): Future[Unit] = { + def writeAsync(path: Path, content: String, options: OpenOptions, compressPayload: Boolean = false): Future[Unit] = asyncCommand(ioCommandBuilder.writeCommand(path, content, options, compressPayload)) - } - def sizeAsync(path: Path): Future[Long] = { + def sizeAsync(path: Path): Future[Long] = asyncCommand(ioCommandBuilder.sizeCommand(path)) - } - def hashAsync(path: Path): Future[String] = { + def hashAsync(path: Path): Future[String] = asyncCommand(ioCommandBuilder.hashCommand(path)) - } - def deleteAsync(path: Path, swallowIoExceptions: Boolean = false): Future[Unit] = { + def deleteAsync(path: Path, swallowIoExceptions: Boolean = false): Future[Unit] = asyncCommand(ioCommandBuilder.deleteCommand(path, swallowIoExceptions)) - } - def existsAsync(path: Path): Future[Boolean] = { + def existsAsync(path: Path): Future[Boolean] = asyncCommand(ioCommandBuilder.existsCommand(path)) - } - def readLinesAsync(path: Path): Future[Iterable[String]] = { + def readLinesAsync(path: Path): Future[Iterable[String]] = asyncCommand(ioCommandBuilder.readLines(path)) - } - def isDirectory(path: Path): Future[Boolean] = { + def isDirectory(path: Path): Future[Boolean] = asyncCommand(ioCommandBuilder.isDirectoryCommand(path)) - } - def copyAsync(src: Path, dest: Path): Future[Unit] = { + def copyAsync(src: Path, dest: Path): Future[Unit] = // Allow for a much larger timeout for copies, as large files can take a while (even on gcs, if they are in different locations...) asyncCommand(ioCommandBuilder.copyCommand(src, dest), AsyncIo.copyTimeout) - } } diff --git a/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala b/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala index 6b4a7763d2a..c7bb7927a00 100644 --- a/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala +++ b/core/src/main/scala/cromwell/core/io/AsyncIoFunctions.scala @@ -3,6 +3,7 @@ package cromwell.core.io import wom.expression.IoFunctionSet trait AsyncIoFunctions { this: IoFunctionSet => + /** * Used to perform io functions asynchronously through the ioActorProxy */ diff --git a/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala b/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala index cf79d06cd2b..59eed184e45 100644 --- a/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala +++ b/core/src/main/scala/cromwell/core/io/CorePathFunctionSet.scala @@ -8,7 +8,9 @@ import wom.expression.{IoFunctionSet, PathFunctionSet} import scala.util.Try class WorkflowCorePathFunctionSet(override val pathBuilders: PathBuilders) extends PathFunctionSet with PathFactory { - private def fail(name: String) = throw new UnsupportedOperationException(s"$name is not implemented at the workflow level") + private def fail(name: String) = throw new UnsupportedOperationException( + s"$name is not implemented at the workflow level" + ) override def sibling(of: String, path: String): String = buildPath(of).sibling(path).pathAsString override def isAbsolute(path: String): Boolean = Try(buildPath(path)).map(_.isAbsolute).toOption.contains(true) override def name(path: String) = buildPath(path).name @@ -19,8 +21,10 @@ class WorkflowCorePathFunctionSet(override val pathBuilders: PathBuilders) exten override def stderr: String = fail("stderr") } -class CallCorePathFunctionSet(pathBuilders: PathBuilders, callContext: CallContext) extends WorkflowCorePathFunctionSet(pathBuilders) { - override def relativeToHostCallRoot(path: String) = if (isAbsolute(path)) path else callContext.root.resolve(path).pathAsString +class CallCorePathFunctionSet(pathBuilders: PathBuilders, callContext: CallContext) + extends WorkflowCorePathFunctionSet(pathBuilders) { + override def relativeToHostCallRoot(path: String) = + if (isAbsolute(path)) path else callContext.root.resolve(path).pathAsString override def stdout = callContext.standardPaths.output.pathAsString override def stderr = callContext.standardPaths.error.pathAsString } @@ -29,7 +33,6 @@ trait WorkflowCorePathFunctions extends { this: IoFunctionSet with PathFactory = override lazy val pathFunctions = new WorkflowCorePathFunctionSet(pathBuilders) } - trait CallCorePathFunctions extends { this: IoFunctionSet with PathFactory => def callContext: CallContext override lazy val pathFunctions = new CallCorePathFunctionSet(pathBuilders, callContext) diff --git a/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala b/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala index bb5e5eec973..44451a1e791 100644 --- a/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala +++ b/core/src/main/scala/cromwell/core/io/DefaultIoCommand.scala @@ -5,13 +5,13 @@ import cromwell.core.io.IoContentAsStringCommand.IoReadOptions import cromwell.core.path.Path object DefaultIoCommand { - case class DefaultIoCopyCommand(override val source: Path, - override val destination: Path, - ) extends IoCopyCommand(source, destination) { + case class DefaultIoCopyCommand(override val source: Path, override val destination: Path) + extends IoCopyCommand(source, destination) { override def commandDescription: String = s"DefaultIoCopyCommand source '$source' destination '$destination'" } - case class DefaultIoContentAsStringCommand(override val file: Path, override val options: IoReadOptions) extends IoContentAsStringCommand(file, options) { + case class DefaultIoContentAsStringCommand(override val file: Path, override val options: IoReadOptions) + extends IoContentAsStringCommand(file, options) { override def commandDescription: String = s"DefaultIoContentAsStringCommand file '$file' options '$options'" } @@ -22,18 +22,24 @@ object DefaultIoCommand { case class DefaultIoWriteCommand(override val file: Path, override val content: String, override val openOptions: OpenOptions, - override val compressPayload: Boolean) extends IoWriteCommand( - file, content, openOptions, compressPayload - ) { + override val compressPayload: Boolean + ) extends IoWriteCommand( + file, + content, + openOptions, + compressPayload + ) { override def commandDescription: String = s"DefaultIoWriteCommand file '$file' content length " + s"'${content.length}' openOptions '$openOptions' compressPayload '$compressPayload'" } - case class DefaultIoDeleteCommand(override val file: Path, - override val swallowIOExceptions: Boolean) extends IoDeleteCommand( - file, swallowIOExceptions - ) { - override def commandDescription: String = s"DefaultIoDeleteCommand file '$file' swallowIOExceptions '$swallowIOExceptions'" + case class DefaultIoDeleteCommand(override val file: Path, override val swallowIOExceptions: Boolean) + extends IoDeleteCommand( + file, + swallowIOExceptions + ) { + override def commandDescription: String = + s"DefaultIoDeleteCommand file '$file' swallowIOExceptions '$swallowIOExceptions'" } case class DefaultIoHashCommand(override val file: Path) extends IoHashCommand(file) { @@ -48,6 +54,10 @@ object DefaultIoCommand { override def commandDescription: String = s"DefaultIoExistsCommand file '$file'" } + case class DefaultIoExistsOrThrowCommand(override val file: Path) extends IoExistsOrThrowCommand(file) { + override def commandDescription: String = s"DefaultIoExistsOrThrowCommand file '$file'" + } + case class DefaultIoReadLinesCommand(override val file: Path) extends IoReadLinesCommand(file) { override def commandDescription: String = s"DefaultIoReadLinesCommand file '$file'" } diff --git a/core/src/main/scala/cromwell/core/io/IoAck.scala b/core/src/main/scala/cromwell/core/io/IoAck.scala index 430b30db792..41066791fe3 100644 --- a/core/src/main/scala/cromwell/core/io/IoAck.scala +++ b/core/src/main/scala/cromwell/core/io/IoAck.scala @@ -8,6 +8,7 @@ import scala.util.{Failure, Success, Try} * @tparam T type of the returned value if success */ sealed trait IoAck[T] { + /** * Original command */ @@ -20,13 +21,12 @@ case class IoSuccess[T](command: IoCommand[T], result: T) extends IoAck[T] { } object IoFailAck { - def unapply(any: Any): Option[(IoCommand[_], Throwable)] = { + def unapply(any: Any): Option[(IoCommand[_], Throwable)] = any match { case f: IoFailAck[_] => Option((f.command, f.failure)) case _ => None } - } } trait IoFailAck[T] extends IoAck[T] { @@ -36,5 +36,7 @@ trait IoFailAck[T] extends IoAck[T] { /** Failure of an unspecified variety. */ case class IoFailure[T](command: IoCommand[T], override val failure: Throwable) extends IoFailAck[T] + /** Specifically read forbidden failure. */ -case class IoReadForbiddenFailure[T](command: IoCommand[T], override val failure: Throwable, forbiddenPath: String) extends IoFailAck[T] +case class IoReadForbiddenFailure[T](command: IoCommand[T], override val failure: Throwable, forbiddenPath: String) + extends IoFailAck[T] diff --git a/core/src/main/scala/cromwell/core/io/IoClientHelper.scala b/core/src/main/scala/cromwell/core/io/IoClientHelper.scala index 26dd0732d33..169e55a496d 100644 --- a/core/src/main/scala/cromwell/core/io/IoClientHelper.scala +++ b/core/src/main/scala/cromwell/core/io/IoClientHelper.scala @@ -13,9 +13,9 @@ trait IoClientHelper extends RobustClientHelper { this: Actor with ActorLogging def ioActor: ActorRef lazy val defaultIoTimeout = RobustClientHelper.DefaultRequestLostTimeout - + protected def config = ConfigFactory.load().as[Config]("system.io.backpressure-backoff") - + override protected def initialBackoff(): Backoff = SimpleExponentialBackoff(config) protected def ioResponseReceive: Receive = { @@ -26,19 +26,16 @@ trait IoClientHelper extends RobustClientHelper { this: Actor with ActorLogging cancelTimeout(context -> ack.command) receive.apply(context -> ack) } - + def ioReceive = robustReceive orElse ioResponseReceive - - def sendIoCommand(ioCommand: IoCommand[_]) = { + + def sendIoCommand(ioCommand: IoCommand[_]) = sendIoCommandWithCustomTimeout(ioCommand, defaultIoTimeout) - } - def sendIoCommandWithCustomTimeout(ioCommand: IoCommand[_], timeout: FiniteDuration) = { + def sendIoCommandWithCustomTimeout(ioCommand: IoCommand[_], timeout: FiniteDuration) = robustSend(ioCommand, ioActor, timeout) - } - def sendIoCommandWithContext[T](ioCommand: IoCommand[_], context: T, timeout: FiniteDuration = defaultIoTimeout) = { + def sendIoCommandWithContext[T](ioCommand: IoCommand[_], context: T, timeout: FiniteDuration = defaultIoTimeout) = robustSend(context -> ioCommand, ioActor, timeout) - } } diff --git a/core/src/main/scala/cromwell/core/io/IoCommand.scala b/core/src/main/scala/cromwell/core/io/IoCommand.scala index a21b6f8cebd..ec90c3b6c5c 100644 --- a/core/src/main/scala/cromwell/core/io/IoCommand.scala +++ b/core/src/main/scala/cromwell/core/io/IoCommand.scala @@ -23,7 +23,7 @@ object IoCommand { .setInitialIntervalMillis((1 second).toMillis.toInt) .setMaxIntervalMillis((5 minutes).toMillis.toInt) .setMultiplier(3L) - .setRandomizationFactor(0.2D) + .setRandomizationFactor(0.2d) .setMaxElapsedTimeMillis((10 minutes).toMillis.toInt) .build() @@ -38,10 +38,16 @@ trait IoCommand[+T] { def commandDescription: String + /** + * IO commands side-effect and/or have exceptions. We don't want that when evaluating identity, e.g. to check presence in cache. + * @return Hash code based on the description, which captures file path & action + */ + override def hashCode(): Int = commandDescription.hashCode + def logIOMsgOverLimit(message: => String): Unit = { val millis: Long = java.time.Duration.between(creation, OffsetDateTime.now).toMillis if (millis > IoCommand.IOCommandWarnLimit.toMillis) { - val seconds = millis / 1000D + val seconds = millis / 1000d /* For now we decided to log this as INFO. In future if needed, we can update this to WARN. @@ -53,8 +59,10 @@ trait IoCommand[+T] { (https://github.com/broadinstitute/firecloud-develop/blob/c77e0f371be0aac545e204f1a134cc6f8ef3c301/run-context/live/configs/cromwell/app.env.ctmpl#L42-L51) - Logback manual (http://logback.qos.ch/manual/index.html) */ - IoCommand.logger.info(f"(IO-$uuid) '$message' is over 5 minutes. It was running for " + - f"$seconds%,.3f seconds. IO command description: '$commandDescription'") + IoCommand.logger.info( + f"(IO-$uuid) '$message' is over 5 minutes. It was running for " + + f"$seconds%,.3f seconds. IO command description: '$commandDescription'" + ) } } @@ -76,7 +84,9 @@ trait IoCommand[+T] { } def failReadForbidden[S >: T](failure: Throwable, forbiddenPath: String): IoReadForbiddenFailure[S] = { - logIOMsgOverLimit(s"IOCommand.failReadForbidden '${failure.toPrettyElidedString(limit = 1000)}' path '$forbiddenPath'") + logIOMsgOverLimit( + s"IOCommand.failReadForbidden '${failure.toPrettyElidedString(limit = 1000)}' path '$forbiddenPath'" + ) IoReadForbiddenFailure(this, failure, forbiddenPath) } @@ -112,7 +122,9 @@ object IoContentAsStringCommand { /** * Read file as a string (load the entire content in memory) */ -abstract class IoContentAsStringCommand(val file: Path, val options: IoReadOptions = IoReadOptions(None, failOnOverflow = false)) extends SingleFileIoCommand[String] { +abstract class IoContentAsStringCommand(val file: Path, + val options: IoReadOptions = IoReadOptions(None, failOnOverflow = false) +) extends SingleFileIoCommand[String] { override def toString = s"read content of ${file.pathAsString}" override lazy val name = "read" } @@ -132,7 +144,8 @@ abstract class IoSizeCommand(val file: Path) extends SingleFileIoCommand[Long] { abstract class IoWriteCommand(val file: Path, val content: String, val openOptions: OpenOptions, - val compressPayload: Boolean) extends SingleFileIoCommand[Unit] { + val compressPayload: Boolean +) extends SingleFileIoCommand[Unit] { override def toString = s"write to ${file.pathAsString}" override lazy val name = "write" } @@ -169,6 +182,14 @@ abstract class IoExistsCommand(val file: Path) extends SingleFileIoCommand[Boole override lazy val name = "exist" } +/** + * Check whether a file exists but throw an exception if it doesn't + */ +abstract class IoExistsOrThrowCommand(val file: Path) extends SingleFileIoCommand[Boolean] { + override def toString = s"Throw error if ${file.pathAsString} does not exist" + override lazy val name = "exist" +} + /** * Return the lines of a file in a collection */ diff --git a/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala b/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala index 43a6f5864b0..ae5ef05e252 100644 --- a/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala +++ b/core/src/main/scala/cromwell/core/io/IoCommandBuilder.scala @@ -21,18 +21,17 @@ abstract class PartialIoCommandBuilder { def hashCommand: PartialFunction[Path, Try[IoHashCommand]] = PartialFunction.empty def touchCommand: PartialFunction[Path, Try[IoTouchCommand]] = PartialFunction.empty def existsCommand: PartialFunction[Path, Try[IoExistsCommand]] = PartialFunction.empty + def existsOrThrowCommand: PartialFunction[Path, Try[IoExistsOrThrowCommand]] = PartialFunction.empty def isDirectoryCommand: PartialFunction[Path, Try[IoIsDirectoryCommand]] = PartialFunction.empty def readLinesCommand: PartialFunction[Path, Try[IoReadLinesCommand]] = PartialFunction.empty } object IoCommandBuilder { - def apply(partialBuilders: PartialIoCommandBuilder*): IoCommandBuilder = { + def apply(partialBuilders: PartialIoCommandBuilder*): IoCommandBuilder = new IoCommandBuilder(partialBuilders.toList) - } - def apply: IoCommandBuilder = { + def apply: IoCommandBuilder = new IoCommandBuilder(List.empty) - } } /** @@ -49,56 +48,61 @@ class IoCommandBuilder(partialBuilders: List[PartialIoCommandBuilder] = List.emp // Find the first partialBuilder for which the partial function is defined, or use the default private def buildOrDefault[A, B](builder: PartialIoCommandBuilder => PartialFunction[A, Try[B]], params: A, - default: => B): Try[B] = { - partialBuilders.to(LazyList).map(builder(_).lift(params)).collectFirst({ - case Some(command) => command - }).getOrElse(Try(default)) - } + default: => B + ): Try[B] = + partialBuilders + .to(LazyList) + .map(builder(_).lift(params)) + .collectFirst { case Some(command) => + command + } + .getOrElse(Try(default)) def contentAsStringCommand(path: Path, maxBytes: Option[Int], - failOnOverflow: Boolean): Try[IoContentAsStringCommand] = { - buildOrDefault(_.contentAsStringCommand, (path, maxBytes, failOnOverflow), DefaultIoContentAsStringCommand(path, IoReadOptions(maxBytes, failOnOverflow))) - } + failOnOverflow: Boolean + ): Try[IoContentAsStringCommand] = + buildOrDefault(_.contentAsStringCommand, + (path, maxBytes, failOnOverflow), + DefaultIoContentAsStringCommand(path, IoReadOptions(maxBytes, failOnOverflow)) + ) def writeCommand(path: Path, content: String, options: OpenOptions, - compressPayload: Boolean = false): Try[IoWriteCommand] = { - buildOrDefault(_.writeCommand, (path, content, options, compressPayload), DefaultIoWriteCommand(path, content, options, compressPayload)) - } - - def sizeCommand(path: Path): Try[IoSizeCommand] = { + compressPayload: Boolean = false + ): Try[IoWriteCommand] = + buildOrDefault(_.writeCommand, + (path, content, options, compressPayload), + DefaultIoWriteCommand(path, content, options, compressPayload) + ) + + def sizeCommand(path: Path): Try[IoSizeCommand] = buildOrDefault(_.sizeCommand, path, DefaultIoSizeCommand(path)) - } - def deleteCommand(path: Path, swallowIoExceptions: Boolean = true): Try[IoDeleteCommand] = { + def deleteCommand(path: Path, swallowIoExceptions: Boolean = true): Try[IoDeleteCommand] = buildOrDefault(_.deleteCommand, (path, swallowIoExceptions), DefaultIoDeleteCommand(path, swallowIoExceptions)) - } - def copyCommand(src: Path, dest: Path): Try[IoCopyCommand] = { + def copyCommand(src: Path, dest: Path): Try[IoCopyCommand] = buildOrDefault(_.copyCommand, (src, dest), DefaultIoCopyCommand(src, dest)) - } - def hashCommand(file: Path): Try[IoHashCommand] = { + def hashCommand(file: Path): Try[IoHashCommand] = buildOrDefault(_.hashCommand, file, DefaultIoHashCommand(file)) - } - def touchCommand(file: Path): Try[IoTouchCommand] = { + def touchCommand(file: Path): Try[IoTouchCommand] = buildOrDefault(_.touchCommand, file, DefaultIoTouchCommand(file)) - } - def existsCommand(file: Path): Try[IoExistsCommand] = { + def existsCommand(file: Path): Try[IoExistsCommand] = buildOrDefault(_.existsCommand, file, DefaultIoExistsCommand(file)) - } - def isDirectoryCommand(file: Path): Try[IoIsDirectoryCommand] = { + def existsOrThrowCommand(file: Path): Try[IoExistsOrThrowCommand] = + buildOrDefault(_.existsOrThrowCommand, file, DefaultIoExistsOrThrowCommand(file)) + + def isDirectoryCommand(file: Path): Try[IoIsDirectoryCommand] = buildOrDefault(_.isDirectoryCommand, file, DefaultIoIsDirectoryCommand(file)) - } - def readLines(file: Path): Try[IoReadLinesCommand] = { + def readLines(file: Path): Try[IoReadLinesCommand] = buildOrDefault(_.readLinesCommand, file, DefaultIoReadLinesCommand(file)) - } } /** diff --git a/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala b/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala index 15097a92a99..e9a8a8934bc 100644 --- a/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala +++ b/core/src/main/scala/cromwell/core/io/IoPromiseProxyActor.scala @@ -10,6 +10,8 @@ import scala.concurrent.duration.FiniteDuration object IoPromiseProxyActor { case class IoCommandWithPromise[A](ioCommand: IoCommand[A], timeout: FiniteDuration = defaultTimeout) { val promise = Promise[A]() + + override def hashCode(): Int = ioCommand.hashCode() } def props(ioActor: ActorRef) = Props(new IoPromiseProxyActor(ioActor)) } @@ -23,17 +25,15 @@ object IoPromiseProxyActor { class IoPromiseProxyActor(override val ioActor: ActorRef) extends Actor with ActorLogging with IoClientHelper { override def receive = ioReceive orElse actorReceive - def actorReceive: Receive = { - case withPromise: IoCommandWithPromise[_] => - sendIoCommandWithContext(withPromise.ioCommand, withPromise.promise, withPromise.timeout) + def actorReceive: Receive = { case withPromise: IoCommandWithPromise[_] => + sendIoCommandWithContext(withPromise.ioCommand, withPromise.promise, withPromise.timeout) } - override protected def ioResponseReceive: Receive = { - case (promise: Promise[_], ack: IoAck[Any] @unchecked) => - cancelTimeout(promise -> ack.command) - // This is not typesafe and assumes the Promise context is of the same type as the IoAck response. - promise.asInstanceOf[Promise[Any]].complete(ack.toTry) - () + override protected def ioResponseReceive: Receive = { case (promise: Promise[_], ack: IoAck[Any] @unchecked) => + cancelTimeout(promise -> ack.command) + // This is not typesafe and assumes the Promise context is of the same type as the IoAck response. + promise.asInstanceOf[Promise[Any]].complete(ack.toTry) + () } override def onTimeout(message: Any, to: ActorRef): Unit = message match { diff --git a/core/src/main/scala/cromwell/core/io/Throttle.scala b/core/src/main/scala/cromwell/core/io/Throttle.scala index 88246f1d88f..aaa5b5d30f5 100644 --- a/core/src/main/scala/cromwell/core/io/Throttle.scala +++ b/core/src/main/scala/cromwell/core/io/Throttle.scala @@ -11,11 +11,10 @@ case class Throttle(elements: Int, per: FiniteDuration, maximumBurst: Int) { } object Throttle { - implicit val throttleOptionValueReader: ValueReader[Option[Throttle]] = (config: Config, path: String) => { + implicit val throttleOptionValueReader: ValueReader[Option[Throttle]] = (config: Config, path: String) => config.getAs[Config](path) map { throttleConfig => val elements = throttleConfig.as[Int]("number-of-requests") val per = throttleConfig.as[FiniteDuration]("per") Throttle(elements, per, elements) } - } } diff --git a/core/src/main/scala/cromwell/core/labels/Label.scala b/core/src/main/scala/cromwell/core/labels/Label.scala index 759300da5b1..616298840b7 100644 --- a/core/src/main/scala/cromwell/core/labels/Label.scala +++ b/core/src/main/scala/cromwell/core/labels/Label.scala @@ -12,21 +12,19 @@ object Label { val MaxLabelLength = 255 val LabelExpectationsMessage = s"A Label key must be non-empty." - def validateLabelKey(s: String): ErrorOr[String] = { + def validateLabelKey(s: String): ErrorOr[String] = (s.length >= 1, s.length <= MaxLabelLength) match { case (true, true) => s.validNel case (false, _) => s"Invalid label: `$s` can't be empty".invalidNel case (_, false) => s"Invalid label: `$s` is ${s.length} characters. The maximum is $MaxLabelLength.".invalidNel } - } - def validateLabelValue(s: String): ErrorOr[String] = { + def validateLabelValue(s: String): ErrorOr[String] = if (s.length <= MaxLabelLength) { s.validNel } else { s"Invalid label: `$s` is ${s.length} characters. The maximum is $MaxLabelLength.".invalidNel } - } def validateLabel(key: String, value: String): ErrorOr[Label] = { val validatedKey = validateLabelKey(key) @@ -35,7 +33,6 @@ object Label { (validatedKey, validatedValue) mapN Label.apply } - def apply(key: String, value: String) = { + def apply(key: String, value: String) = new Label(key, value) {} - } } diff --git a/core/src/main/scala/cromwell/core/labels/Labels.scala b/core/src/main/scala/cromwell/core/labels/Labels.scala index 5499fa5b2e8..148a0a7926d 100644 --- a/core/src/main/scala/cromwell/core/labels/Labels.scala +++ b/core/src/main/scala/cromwell/core/labels/Labels.scala @@ -19,13 +19,11 @@ case class Labels(value: Vector[Label]) { } object Labels { - def apply(values: (String, String)*): Labels = { + def apply(values: (String, String)*): Labels = Labels(values.toVector map (Label.apply _).tupled) - } - def validateMapOfLabels(labels: Map[String, String]): ErrorOr[Labels] = { + def validateMapOfLabels(labels: Map[String, String]): ErrorOr[Labels] = labels.toVector traverse { Label.validateLabel _ }.tupled map Labels.apply - } def empty = Labels(Vector.empty) } diff --git a/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala b/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala index dcf62bc0b28..47eb951199a 100644 --- a/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala +++ b/core/src/main/scala/cromwell/core/logging/EnhancedDateConverter.scala @@ -25,9 +25,9 @@ class EnhancedDateConverter extends DateConverter { cachingDateFormatterProtected = Option(getFirstOption) match { case Some(CoreConstants.ISO8601_STR) | None => new CachingDateFormatter(CoreConstants.ISO8601_PATTERN) case Some(datePattern) => - try { + try new CachingDateFormatter(datePattern) - } catch { + catch { case e: IllegalArgumentException => addWarn("Could not instantiate SimpleDateFormat with pattern " + datePattern, e) // default to the ISO8601 format @@ -35,8 +35,7 @@ class EnhancedDateConverter extends DateConverter { } } // if the option list contains a TZ option, then set it. - Option(getOptionList) - .toList + Option(getOptionList).toList .flatMap(_.asScala) .drop(1) .headOption diff --git a/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala b/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala index 0999ec18055..6f9e5f2fdfb 100644 --- a/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/EnhancedSlf4jLogger.scala @@ -3,6 +3,7 @@ package cromwell.core.logging import akka.event.slf4j.Slf4jLogger class EnhancedSlf4jLogger extends Slf4jLogger { + /** * Format the timestamp as a simple long. Allows the akkaTimestamp to be retrieved later from the MDC by custom * converters. diff --git a/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala b/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala index 1dc10fab46d..beff484f941 100644 --- a/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala +++ b/core/src/main/scala/cromwell/core/logging/JavaLoggingBridge.scala @@ -8,6 +8,7 @@ import org.slf4j.bridge.SLF4JBridgeHandler import scala.jdk.CollectionConverters._ object JavaLoggingBridge { + /** * Replace java.util.logging with SLF4J while ensuring Logback is configured with a LevelChangePropogator. * diff --git a/core/src/main/scala/cromwell/core/logging/JobLogger.scala b/core/src/main/scala/cromwell/core/logging/JobLogger.scala index 37ab6cfa2da..4af851124cb 100644 --- a/core/src/main/scala/cromwell/core/logging/JobLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/JobLogger.scala @@ -28,14 +28,14 @@ class JobLogger(loggerName: String, rootWorkflowIdForLogging: RootWorkflowId, jobTag: String, akkaLogger: Option[LoggingAdapter] = None, - otherLoggers: Set[Logger] = Set.empty[Logger]) - extends WorkflowLogger( - loggerName = loggerName, - workflowId = workflowIdForLogging, - rootWorkflowId = rootWorkflowIdForLogging, - akkaLogger = akkaLogger, - otherLoggers = otherLoggers - ) { + otherLoggers: Set[Logger] = Set.empty[Logger] +) extends WorkflowLogger( + loggerName = loggerName, + workflowId = workflowIdForLogging, + rootWorkflowId = rootWorkflowIdForLogging, + akkaLogger = akkaLogger, + otherLoggers = otherLoggers + ) { override def tag = s"$loggerName [UUID(${workflowIdForLogging.shortString})$jobTag]" } diff --git a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala index 5af71c7099c..3f9044aeaa3 100644 --- a/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala +++ b/core/src/main/scala/cromwell/core/logging/LoggerWrapper.scala @@ -23,9 +23,8 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { * * https://github.com/qos-ch/slf4j/blob/v_1.7.30/slf4j-simple/src/main/java/org/slf4j/impl/SimpleLogger.java#L293-L295 */ - private def format(msg: String, throwable: Throwable): String = { + private def format(msg: String, throwable: Throwable): String = format(msg) + "\n" + ExceptionUtils.getStackTrace(throwable) - } /** * Passes a formatted string to akka similar to slf4j's SimpleLogger @@ -113,7 +112,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { lazy val formatted: String = format(pattern) varargsAkkaLog(Logging.ErrorLevel, pattern, arguments) - slf4jLoggers.foreach(_.error(formatted, arguments:_*)) + slf4jLoggers.foreach(_.error(formatted, arguments: _*)) } override def error(pattern: String, arg: Any): Unit = { @@ -130,10 +129,9 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.error(formatted, arg1, arg2: Any)) } - def error(t: Throwable, pattern: String, arguments: Any*): Unit = { + def error(t: Throwable, pattern: String, arguments: Any*): Unit = // slf4j extracts the last variable argument as a throwable. error(pattern, (arguments :+ t).map(_.asInstanceOf[AnyRef]): _*) - } override def debug(msg: String): Unit = { lazy val formatted: String = format(msg) @@ -153,7 +151,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { lazy val formatted: String = format(pattern) varargsAkkaLog(Logging.DebugLevel, pattern, arguments) - slf4jLoggers.foreach(_.debug(formatted, arguments:_*)) + slf4jLoggers.foreach(_.debug(formatted, arguments: _*)) } override def debug(pattern: String, argument: Any): Unit = { @@ -170,25 +168,20 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.debug(formatted, arg1, arg2: Any)) } - override def trace(msg: String): Unit = { + override def trace(msg: String): Unit = slf4jLoggers.foreach(_.trace(format(msg))) - } - override def trace(msg: String, t: Throwable): Unit = { + override def trace(msg: String, t: Throwable): Unit = slf4jLoggers.foreach(_.trace(format(msg), t)) - } - override def trace(pattern: String, arguments: AnyRef*): Unit = { - slf4jLoggers.foreach(_.trace(format(pattern), arguments:_*)) - } + override def trace(pattern: String, arguments: AnyRef*): Unit = + slf4jLoggers.foreach(_.trace(format(pattern), arguments: _*)) - override def trace(pattern: String, arg: Any): Unit = { + override def trace(pattern: String, arg: Any): Unit = slf4jLoggers.foreach(_.trace(format(pattern), arg)) - } - override def trace(pattern: String, arg1: Any, arg2: Any): Unit = { + override def trace(pattern: String, arg1: Any, arg2: Any): Unit = slf4jLoggers.foreach(_.trace(format(pattern), arg1, arg2: Any)) - } override def info(msg: String): Unit = { lazy val formatted: String = format(msg) @@ -208,7 +201,7 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { lazy val formatted: String = format(pattern) varargsAkkaLog(Logging.InfoLevel, pattern, arguments) - slf4jLoggers.foreach(_.info(formatted, arguments:_*)) + slf4jLoggers.foreach(_.info(formatted, arguments: _*)) } override def info(pattern: String, arg: Any): Unit = { @@ -225,14 +218,24 @@ abstract class LoggerWrapper extends MarkerIgnoringBase { slf4jLoggers.foreach(_.info(formatted, arg1, arg2: Any)) } - override def isErrorEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isErrorEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isInfoEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isInfoEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isDebugEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isDebugEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isTraceEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isTraceEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) - override def isWarnEnabled: Boolean = throw new UnsupportedOperationException("This logger wraps an arbitrary set of loggers that can each have a different level enabled.") + override def isWarnEnabled: Boolean = throw new UnsupportedOperationException( + "This logger wraps an arbitrary set of loggers that can each have a different level enabled." + ) } diff --git a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala index 94028b39407..d404cb36336 100644 --- a/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala +++ b/core/src/main/scala/cromwell/core/logging/WorkflowLogger.scala @@ -49,41 +49,40 @@ object WorkflowLogger { https://github.com/qos-ch/logback/commit/77128a003a7fd7e8bd7a6ddb12da7a65cf296593#diff-f8cd32379a53986c2e70e2abe86fa0faR145 */ private def makeSynchronizedFileLogger(path: Path, level: Level, ctx: LoggerContext, name: String): Logger = - ctx.synchronized { - Option(ctx.exists(name)) match { - case Some(existingLogger) => existingLogger - case None => - val encoder = new PatternLayoutEncoder() - encoder.setPattern("%date %-5level - %msg%n") - encoder.setContext(ctx) - encoder.start() - - val appender = new FileAppender[ILoggingEvent]() - appender.setFile(path.pathAsString) - appender.setEncoder(encoder) - appender.setName(name) - appender.setContext(ctx) - appender.start() - - val fileLogger = ctx.getLogger(name) - fileLogger.addAppender(appender) - fileLogger.setAdditive(false) - fileLogger.setLevel(level) - fileLogger + ctx.synchronized { + Option(ctx.exists(name)) match { + case Some(existingLogger) => existingLogger + case None => + val encoder = new PatternLayoutEncoder() + encoder.setPattern("%date %-5level - %msg%n") + encoder.setContext(ctx) + encoder.start() + + val appender = new FileAppender[ILoggingEvent]() + appender.setFile(path.pathAsString) + appender.setEncoder(encoder) + appender.setName(name) + appender.setContext(ctx) + appender.start() + + val fileLogger = ctx.getLogger(name) + fileLogger.addAppender(appender) + fileLogger.setAdditive(false) + fileLogger.setLevel(level) + fileLogger + } } - } case class WorkflowLogConfiguration(dir: Path, temporary: Boolean) private val conf = ConfigFactory.load() - val workflowLogConfiguration: Option[WorkflowLogConfiguration] = { + val workflowLogConfiguration: Option[WorkflowLogConfiguration] = for { workflowConfig <- conf.as[Option[Config]]("workflow-options") dir <- workflowConfig.as[Option[String]]("workflow-log-dir") if !dir.isEmpty temporary <- workflowConfig.as[Option[Boolean]]("workflow-log-temporary") orElse Option(true) } yield WorkflowLogConfiguration(DefaultPathBuilder.get(dir).toAbsolutePath, temporary) - } val isEnabled = workflowLogConfiguration.isDefined val isTemporary = workflowLogConfiguration exists { @@ -111,8 +110,8 @@ class WorkflowLogger(loggerName: String, workflowId: PossiblyNotRootWorkflowId, rootWorkflowId: RootWorkflowId, override val akkaLogger: Option[LoggingAdapter], - otherLoggers: Set[Logger] = Set.empty[Logger]) - extends LoggerWrapper { + otherLoggers: Set[Logger] = Set.empty[Logger] +) extends LoggerWrapper { override def getName = loggerName @@ -137,7 +136,8 @@ class WorkflowLogger(loggerName: String, import WorkflowLogger._ lazy val workflowLogPath = workflowLogConfiguration.map(workflowLogConfigurationActual => - workflowLogConfigurationActual.dir.createPermissionedDirectories() / s"workflow.$rootWorkflowId.log") + workflowLogConfigurationActual.dir.createPermissionedDirectories() / s"workflow.$rootWorkflowId.log" + ) lazy val fileLogger = workflowLogPath match { case Some(path) => makeFileLogger(path, Level.toLevel(sys.props.getOrElse("LOG_LEVEL", "debug"))) diff --git a/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala index f0a6464f12e..ba300da9b83 100644 --- a/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala +++ b/core/src/main/scala/cromwell/core/path/BetterFileMethods.scala @@ -27,9 +27,9 @@ trait BetterFileMethods { import BetterFileMethods._ - private final def newPath(file: better.files.File): Path = newPath(file.path) + final private def newPath(file: better.files.File): Path = newPath(file.path) - private final def newPathOrNull(file: better.files.File): Path = Option(file).map(newPath).orNull + final private def newPathOrNull(file: better.files.File): Path = Option(file).map(newPath).orNull final def toJava: JFile = betterFile.toJava @@ -44,8 +44,10 @@ trait BetterFileMethods { final def extension: Option[String] = betterFile.extension - final def extension(includeDot: Boolean = true, includeAll: Boolean = false, - toLowerCase: Boolean = true): Option[String] = + final def extension(includeDot: Boolean = true, + includeAll: Boolean = false, + toLowerCase: Boolean = true + ): Option[String] = betterFile.extension(includeDot, includeAll, toLowerCase) final def hasExtension: Boolean = betterFile.hasExtension @@ -60,16 +62,17 @@ trait BetterFileMethods { final def /(child: String): Path = newPath(betterFile./(child)) - final def createChild(child: String, asDirectory: Boolean = false) - (implicit attributes: Attributes = Attributes.default, - linkOptions: LinkOptions = LinkOptions.default): Path = + final def createChild(child: String, asDirectory: Boolean = false)(implicit + attributes: Attributes = Attributes.default, + linkOptions: LinkOptions = LinkOptions.default + ): Path = newPath(betterFile.createChild(child, asDirectory)(attributes, linkOptions)) - final def createIfNotExists(asDirectory: Boolean = false, createParents: Boolean = false) - (implicit attributes: Attributes = Attributes.default, - linkOptions: LinkOptions = LinkOptions.default): Path = { + final def createIfNotExists(asDirectory: Boolean = false, createParents: Boolean = false)(implicit + attributes: Attributes = Attributes.default, + linkOptions: LinkOptions = LinkOptions.default + ): Path = newPath(betterFile.createIfNotExists(asDirectory, createParents)(attributes, linkOptions)) - } final def exists(implicit linkOptions: LinkOptions = LinkOptions.default): Boolean = betterFile.exists(linkOptions) @@ -108,15 +111,17 @@ trait BetterFileMethods { final def lines(implicit charset: Charset = DefaultCharset): Iterable[String] = betterFile.lines(charset) - final def lineIterator(implicit charset: Charset= DefaultCharset): Iterator[String] = betterFile.lineIterator(charset) + final def lineIterator(implicit charset: Charset = DefaultCharset): Iterator[String] = + betterFile.lineIterator(charset) - final def tokens(splitter: StringSplitter = StringSplitter.Default) - (implicit charset: Charset = DefaultCharset): Iterator[String] = + final def tokens(splitter: StringSplitter = StringSplitter.Default)(implicit + charset: Charset = DefaultCharset + ): Iterator[String] = betterFile.tokens(splitter)(charset) final def contentAsString(implicit charset: Charset = DefaultCharset): String = betterFile.contentAsString(charset) - final def `!`(implicit charset: Charset= DefaultCharset): String = betterFile.contentAsString(charset) + final def `!`(implicit charset: Charset = DefaultCharset): String = betterFile.contentAsString(charset) final def printLines(lines: Iterator[Any])(implicit openOptions: OpenOptions = OpenOptions.append): this.type = { betterFile.printLines(lines)(openOptions) @@ -173,33 +178,37 @@ trait BetterFileMethods { this } - final def writeText(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def writeText( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.writeText(text)(openOptions, charset) this } - final def write(text: String) - (implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def write( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.write(text)(openOptions, charset) this } - final def overwrite(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def overwrite( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.overwrite(text)(openOptions, charset) this } - final def <(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def <( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.write(text)(openOptions, charset) this } - final def `>:`(text: String)(implicit openOptions: OpenOptions = OpenOptions.default, - charset: Charset = DefaultCharset): this.type = { + final def `>:`( + text: String + )(implicit openOptions: OpenOptions = OpenOptions.default, charset: Charset = DefaultCharset): this.type = { betterFile.write(text)(openOptions, charset) this } @@ -221,12 +230,16 @@ trait BetterFileMethods { final def bufferedReader(implicit charset: Charset = DefaultCharset): Dispose[BufferedReader] = betterFile.bufferedReader(charset) - final def newBufferedWriter(implicit charset: Charset = DefaultCharset, - openOptions: OpenOptions = OpenOptions.default): BufferedWriter = + final def newBufferedWriter(implicit + charset: Charset = DefaultCharset, + openOptions: OpenOptions = OpenOptions.default + ): BufferedWriter = betterFile.newBufferedWriter(charset, openOptions) - final def bufferedWriter(implicit charset: Charset = DefaultCharset, - openOptions: OpenOptions = OpenOptions.default): Dispose[BufferedWriter] = + final def bufferedWriter(implicit + charset: Charset = DefaultCharset, + openOptions: OpenOptions = OpenOptions.default + ): Dispose[BufferedWriter] = betterFile.bufferedWriter(charset, openOptions) final def newFileReader: FileReader = betterFile.newFileReader @@ -237,12 +250,14 @@ trait BetterFileMethods { final def fileWriter(append: Boolean = false): Dispose[FileWriter] = betterFile.fileWriter(append) - final def newPrintWriter(autoFlush: Boolean = false) - (implicit openOptions: OpenOptions = OpenOptions.default): PrintWriter = + final def newPrintWriter(autoFlush: Boolean = false)(implicit + openOptions: OpenOptions = OpenOptions.default + ): PrintWriter = betterFile.newPrintWriter(autoFlush) - final def printWriter(autoFlush: Boolean = false) - (implicit openOptions: OpenOptions = OpenOptions.default): Dispose[PrintWriter] = + final def printWriter(autoFlush: Boolean = false)(implicit + openOptions: OpenOptions = OpenOptions.default + ): Dispose[PrintWriter] = betterFile.printWriter(autoFlush) final def newInputStream(implicit openOptions: OpenOptions = OpenOptions.default): InputStream = @@ -251,12 +266,14 @@ trait BetterFileMethods { final def inputStream(implicit openOptions: OpenOptions = OpenOptions.default): Dispose[InputStream] = betterFile.inputStream(openOptions) - final def newScanner(splitter: StringSplitter = StringSplitter.Default) - (implicit charset: Charset = DefaultCharset): Scanner = + final def newScanner(splitter: StringSplitter = StringSplitter.Default)(implicit + charset: Charset = DefaultCharset + ): Scanner = betterFile.newScanner(splitter)(charset) - final def scanner(splitter: StringSplitter = StringSplitter.Default) - (implicit charset: Charset = DefaultCharset): Dispose[Scanner] = + final def scanner(splitter: StringSplitter = StringSplitter.Default)(implicit + charset: Charset = DefaultCharset + ): Dispose[Scanner] = betterFile.scanner(splitter)(charset) final def newOutputStream(implicit openOptions: OpenOptions = OpenOptions.default): OutputStream = @@ -265,19 +282,25 @@ trait BetterFileMethods { final def outputStream(implicit openOptions: OpenOptions = OpenOptions.default): Dispose[OutputStream] = betterFile.outputStream(openOptions) - final def newFileChannel(implicit openOptions: OpenOptions = OpenOptions.default, - attributes: Attributes = Attributes.default): FileChannel = + final def newFileChannel(implicit + openOptions: OpenOptions = OpenOptions.default, + attributes: Attributes = Attributes.default + ): FileChannel = betterFile.newFileChannel(openOptions, attributes) - final def fileChannel(implicit openOptions: OpenOptions = OpenOptions.default, - attributes: Attributes = Attributes.default): Dispose[FileChannel] = + final def fileChannel(implicit + openOptions: OpenOptions = OpenOptions.default, + attributes: Attributes = Attributes.default + ): Dispose[FileChannel] = betterFile.fileChannel(openOptions, attributes) - final def newAsynchronousFileChannel(implicit openOptions: OpenOptions = OpenOptions.default): - AsynchronousFileChannel = betterFile.newAsynchronousFileChannel(openOptions) + final def newAsynchronousFileChannel(implicit + openOptions: OpenOptions = OpenOptions.default + ): AsynchronousFileChannel = betterFile.newAsynchronousFileChannel(openOptions) - final def asynchronousFileChannel(implicit openOptions: OpenOptions = OpenOptions.default): - Dispose[AsynchronousFileChannel] = betterFile.asynchronousFileChannel(openOptions) + final def asynchronousFileChannel(implicit + openOptions: OpenOptions = OpenOptions.default + ): Dispose[AsynchronousFileChannel] = betterFile.asynchronousFileChannel(openOptions) final def digest(algorithmName: String): Array[Byte] = { val messageDigest = MessageDigest.getInstance(algorithmName) @@ -307,8 +330,11 @@ trait BetterFileMethods { final def isHidden: Boolean = betterFile.isHidden - final def isLocked(mode: RandomAccessMode, position: Long = 0L, size: Long = Long.MaxValue, - isShared: Boolean = false): Boolean = betterFile.isLocked(mode, position, size, isShared) + final def isLocked(mode: RandomAccessMode, + position: Long = 0L, + size: Long = Long.MaxValue, + isShared: Boolean = false + ): Boolean = betterFile.isLocked(mode, position, size, isShared) final def isReadLocked(position: Long = 0L, size: Long = Long.MaxValue, isShared: Boolean = false): Boolean = betterFile.isReadLocked(position, size, isShared) @@ -359,7 +385,7 @@ trait BetterFileMethods { } // Conflicts with the legacy cromwell.core.path.Obsolete.PathMethodAliases.getFileName(). Uncomment when that's gone. - //final def apply(permission: PosixFilePermission): Boolean = betterFile.apply(permission) + // final def apply(permission: PosixFilePermission): Boolean = betterFile.apply(permission) final def isOwnerReadable: Boolean = betterFile.isOwnerReadable @@ -416,9 +442,9 @@ trait BetterFileMethods { this } - final def touch(time: Instant = Instant.now()) - (implicit attributes: Attributes = Attributes.default, - linkOptions: LinkOptions = LinkOptions.default): this.type = { + final def touch( + time: Instant = Instant.now() + )(implicit attributes: Attributes = Attributes.default, linkOptions: LinkOptions = LinkOptions.default): this.type = { betterFile.touch(time)(attributes, linkOptions) this } @@ -443,14 +469,16 @@ trait BetterFileMethods { destination } - final def symbolicLinkTo(destination: Path) - (implicit attributes: Attributes = Attributes.default): destination.type = { + final def symbolicLinkTo( + destination: Path + )(implicit attributes: Attributes = Attributes.default): destination.type = { betterFile.symbolicLinkTo(destination.betterFile)(attributes) destination } - final def linkTo(destination: Path, symbolic: Boolean = false) - (implicit attributes: Attributes = Attributes.default): destination.type = { + final def linkTo(destination: Path, symbolic: Boolean = false)(implicit + attributes: Attributes = Attributes.default + ): destination.type = { betterFile.linkTo(destination.betterFile, symbolic)(attributes) destination } @@ -477,13 +505,16 @@ trait BetterFileMethods { this } - final def zipTo(destination: Path, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION) - (implicit charset: Charset = DefaultCharset): destination.type = { + final def zipTo(destination: Path, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit + charset: Charset = DefaultCharset + ): destination.type = { betterFile.zipTo(destination.betterFile, compressionLevel)(charset) destination } - final def zip(compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit charset: Charset = DefaultCharset): Path = + final def zip(compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit + charset: Charset = DefaultCharset + ): Path = newPath(betterFile.zip(compressionLevel)(charset)) final def unzipTo(destination: Path)(implicit charset: Charset = DefaultCharset): destination.type = { @@ -515,9 +546,9 @@ object BetterFileMethods { def cwd: Path = pwd - val `..`: Path => Path = _.parent + val `..` : Path => Path = _.parent - val `.`: Path => Path = identity + val `.` : Path => Path = identity implicit class FileDsl(file: Path) { def /(f: Path => Path): Path = f(file) @@ -561,7 +592,8 @@ object BetterFileMethods { def chgrp(group: String, file: Path): Path = file.setGroup(group) - def chmod(permissions: String, file: Path): Path = file.setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet) + def chmod(permissions: String, file: Path): Path = + file.setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet) def chmod_+(permission: PosixFilePermission, file: Path): Path = file.addPermission(permission) @@ -572,10 +604,12 @@ object BetterFileMethods { def unzip(zipFile: Path)(destination: Path)(implicit charset: Charset = DefaultCharset): destination.type = zipFile.unzipTo(destination)(charset) - def zip(files: better.files.File*)(destination: better.files.File, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION) - (implicit charset: Charset = DefaultCharset): destination.type = { + def zip( + files: better.files.File* + )(destination: better.files.File, compressionLevel: Int = Deflater.DEFAULT_COMPRESSION)(implicit + charset: Charset = DefaultCharset + ): destination.type = destination.zipIn(files.iterator, compressionLevel)(charset) - } } type PathMatcherSyntax = better.files.File.PathMatcherSyntax diff --git a/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala b/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala index 13f577a1718..5136d7baa20 100644 --- a/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala +++ b/core/src/main/scala/cromwell/core/path/CustomRetryParams.scala @@ -11,7 +11,7 @@ object CustomRetryParams { val Default = CustomRetryParams( timeout = Duration.Inf, maxRetries = Option(3), - backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5D), + backoff = SimpleExponentialBackoff(1 seconds, 3 seconds, 1.5d), isTransient = throwableToFalse, isFatal = throwableToFalse ) @@ -23,4 +23,5 @@ case class CustomRetryParams(timeout: Duration, maxRetries: Option[Int], backoff: Backoff, isTransient: Throwable => Boolean, - isFatal: Throwable => Boolean) + isFatal: Throwable => Boolean +) diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala index 4bbefe71d0a..f478d04aa67 100644 --- a/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala +++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilder.scala @@ -19,7 +19,6 @@ case object DefaultPathBuilder extends PathBuilder { val uri = URI.create(UrlEscapers.urlFragmentEscaper().escape(pathAsString)) Option(uri.getScheme) match { case Some("file") | None => - if (pathAsString.startsWith("file://")) { // NOTE: Legacy support for old paths generated as URIs by the old .toRealString val host = Option(uri.getHost) getOrElse "" @@ -44,15 +43,14 @@ case object DefaultPathBuilder extends PathBuilder { def createTempDirectory(prefix: String): DefaultPath = DefaultPath(java.nio.file.Files.createTempDirectory(prefix)) - def createTempFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = { + def createTempFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = parent match { case Some(dir) => dir.createTempFile(prefix, suffix) case _ => DefaultPath(java.nio.file.Files.createTempFile(prefix, suffix)) } - } } -case class DefaultPath private[path](nioPath: NioPath) extends Path { +case class DefaultPath private[path] (nioPath: NioPath) extends Path { override protected def newPath(nioPath: NioPath): DefaultPath = DefaultPath(nioPath) override def pathAsString: String = nioPath.toString diff --git a/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala index 91f05b10ef7..f986dcfa1a5 100644 --- a/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala +++ b/core/src/main/scala/cromwell/core/path/DefaultPathBuilderFactory.scala @@ -2,11 +2,15 @@ package cromwell.core.path import akka.actor.ActorSystem import cromwell.core.WorkflowOptions +import cromwell.core.path.PathBuilderFactory.PriorityDefault import scala.concurrent.{ExecutionContext, Future} case object DefaultPathBuilderFactory extends PathBuilderFactory { - override def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem, ec: ExecutionContext) = Future.successful(DefaultPathBuilder) + override def withOptions(options: WorkflowOptions)(implicit actorSystem: ActorSystem, ec: ExecutionContext) = + Future.successful(DefaultPathBuilder) val name = "local" val tuple = name -> this + + override def priority: Int = PriorityDefault } diff --git a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala index ffe862c2df3..e455867e5e6 100644 --- a/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala +++ b/core/src/main/scala/cromwell/core/path/EvenBetterPathMethods.scala @@ -1,6 +1,6 @@ package cromwell.core.path -import java.io.{BufferedInputStream, BufferedReader, ByteArrayOutputStream, IOException, InputStream, InputStreamReader} +import java.io.{BufferedInputStream, BufferedReader, ByteArrayOutputStream, InputStream, InputStreamReader, IOException} import java.nio.file.{FileAlreadyExistsException, Files} import java.nio.file.attribute.{PosixFilePermission, PosixFilePermissions} import java.util.zip.GZIPOutputStream @@ -33,13 +33,11 @@ trait EvenBetterPathMethods { final def plusSuffix(suffix: String): Path = swapSuffix("", suffix) - final def swapSuffix(oldSuffix: String, newSuffix: String): Path = { + final def swapSuffix(oldSuffix: String, newSuffix: String): Path = sibling(s"${name.stripSuffix(oldSuffix)}$newSuffix") - } - final def createTempFile(prefix: String = "", suffix: String = ""): Path = { + final def createTempFile(prefix: String = "", suffix: String = ""): Path = newPath(java.nio.file.Files.createTempFile(nioPathPrivate, prefix, suffix)) - } def chmod(permissions: String): this.type = { setPermissions(PosixFilePermissions.fromString(permissions).asScala.toSet) @@ -49,18 +47,16 @@ trait EvenBetterPathMethods { // betterFile.symbolicLink calls Files.readSymbolicLink, but then implicitly converts the java.nio.Path returned to a better.File // which calls toAbsolutePath. Consequently, if the path was relative, the current directory is used to make it absolute. // This is not the desired behavior to be able to follow relative symbolic links, so bypass better files method and directly use the java one. - final def symbolicLinkRelative: Option[Path] = { + final def symbolicLinkRelative: Option[Path] = if (betterFile.isSymbolicLink) { Option(newPath(Files.readSymbolicLink(betterFile.path))) } else None - } - final def followSymbolicLinks: Path = { + final def followSymbolicLinks: Path = symbolicLinkRelative match { case Some(target) => parent.resolve(target.followSymbolicLinks) case None => this } - } final def createPermissionedDirectories(): this.type = { if (!exists) { @@ -73,8 +69,7 @@ trait EvenBetterPathMethods { addPermission(PosixFilePermission.OTHERS_READ) addPermission(PosixFilePermission.OTHERS_WRITE) addPermission(PosixFilePermission.OTHERS_EXECUTE) - } - catch { + } catch { // Race condition that's particularly likely with scatters. Ignore. case _: FileAlreadyExistsException => // The GCS filesystem does not support setting permissions and will throw an `UnsupportedOperationException`. @@ -105,7 +100,9 @@ trait EvenBetterPathMethods { byteStream.toByteArray } - def writeContent(content: String)(openOptions: OpenOptions, codec: Codec, compressPayload: Boolean)(implicit ec: ExecutionContext): this.type = { + def writeContent( + content: String + )(openOptions: OpenOptions, codec: Codec, compressPayload: Boolean)(implicit ec: ExecutionContext): this.type = { locally(ec) val contentByteArray = content.getBytes(codec.charSet) writeByteArray { @@ -113,8 +110,8 @@ trait EvenBetterPathMethods { }(openOptions) } - private def fileIoErrorPf[A]: PartialFunction[Throwable, Try[A]] = { - case ex: Throwable => Failure(new IOException(s"Could not read from ${this.pathAsString}: ${ex.getMessage}", ex)) + private def fileIoErrorPf[A]: PartialFunction[Throwable, Try[A]] = { case ex: Throwable => + Failure(new IOException(s"Could not read from ${this.pathAsString}: ${ex.getMessage}", ex)) } /** @@ -122,35 +119,36 @@ trait EvenBetterPathMethods { * The input stream will be closed when this method returns, which means the f function * cannot leak an open stream. */ - def withReader[A](f: BufferedReader => A)(implicit ec: ExecutionContext): A = { + def withReader[A](f: BufferedReader => A)(implicit ec: ExecutionContext): A = // Use an input reader to convert the byte stream to character stream. Buffered reader for efficiency. - tryWithResource(() => new BufferedReader(new InputStreamReader(this.mediaInputStream, Codec.UTF8.name)))(f).recoverWith(fileIoErrorPf).get - } + tryWithResource(() => new BufferedReader(new InputStreamReader(this.mediaInputStream, Codec.UTF8.name)))(f) + .recoverWith(fileIoErrorPf) + .get /** * InputStream's read method reads bytes, whereas InputStreamReader's read method reads characters. * BufferedInputStream can be used to read bytes directly from input stream, without conversion to characters. */ - def withBufferedStream[A](f: BufferedInputStream => A)(implicit ec: ExecutionContext): A = { + def withBufferedStream[A](f: BufferedInputStream => A)(implicit ec: ExecutionContext): A = tryWithResource(() => new BufferedInputStream(this.mediaInputStream))(f).recoverWith(fileIoErrorPf).get - } /** * Returns an Array[Byte] from a Path. Limit the array size to "limit" byte if defined. * @throws IOException if failOnOverflow is true and the file is larger than limit */ - def limitFileContent(limit: Option[Int], failOnOverflow: Boolean)(implicit ec: ExecutionContext): Array[Byte] = withBufferedStream { bufferedStream => - val bytesIterator = Iterator.continually(bufferedStream.read).takeWhile(_ != -1).map(_.toByte) - // Take 1 more than the limit so that we can look at the size and know if it's overflowing - val bytesArray = limit.map(l => bytesIterator.take(l + 1)).getOrElse(bytesIterator).toArray - - limit match { - case Some(l) if failOnOverflow && bytesArray.length > l => - throw new IOException(s"File $this is larger than requested maximum of $l Bytes.") - case Some(l) => bytesArray.take(l) - case _ => bytesArray + def limitFileContent(limit: Option[Int], failOnOverflow: Boolean)(implicit ec: ExecutionContext): Array[Byte] = + withBufferedStream { bufferedStream => + val bytesIterator = Iterator.continually(bufferedStream.read).takeWhile(_ != -1).map(_.toByte) + // Take 1 more than the limit so that we can look at the size and know if it's overflowing + val bytesArray = limit.map(l => bytesIterator.take(l + 1)).getOrElse(bytesIterator).toArray + + limit match { + case Some(l) if failOnOverflow && bytesArray.length > l => + throw new IOException(s"File $this is larger than requested maximum of $l Bytes.") + case Some(l) => bytesArray.take(l) + case _ => bytesArray + } } - } /** * Reads the first limitBytes of a file and makes a String. Prepend with an annotation at the start (to say that this is the diff --git a/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala b/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala index cc1b7f40dde..2b5e36f2086 100644 --- a/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala +++ b/core/src/main/scala/cromwell/core/path/JavaWriterImplicits.scala @@ -4,6 +4,7 @@ import java.io.Writer object JavaWriterImplicits { implicit class FlushingAndClosingWriter(writer: Writer) { + /** Convenience method to flush and close in one shot. */ def flushAndClose() = { writer.flush() diff --git a/core/src/main/scala/cromwell/core/path/NioPathMethods.scala b/core/src/main/scala/cromwell/core/path/NioPathMethods.scala index f5791e50c5a..3ce92dddd93 100644 --- a/core/src/main/scala/cromwell/core/path/NioPathMethods.scala +++ b/core/src/main/scala/cromwell/core/path/NioPathMethods.scala @@ -2,7 +2,6 @@ package cromwell.core.path import java.nio.file.WatchEvent.{Kind, Modifier} import java.nio.file.{LinkOption, WatchKey, WatchService} - import scala.jdk.CollectionConverters._ /** @@ -36,9 +35,9 @@ trait NioPathMethods { final def getNameCount: Int = nioPathPrivate.getNameCount /* This method cannot be used safely because it could fail for valid GcsPaths that are not valid URIs - * See https://github.com/GoogleCloudPlatform/google-cloud-java/issues/1343 + * See https://github.com/GoogleCloudPlatform/google-cloud-java/issues/1343 */ - //final def toUri: URI = nioPathPrivate.toUri + // final def toUri: URI = nioPathPrivate.toUri final def compareTo(other: Path): Int = nioPathPrivate.compareTo(other.nioPathPrivate) @@ -68,4 +67,11 @@ trait NioPathMethods { final def startsWith(other: String): Boolean = nioPathPrivate.startsWith(other) final def toRealPath(options: LinkOption*): Path = newPath(nioPathPrivate.toRealPath(options: _*)) + + /** + * Get a valid path object that resolves symlinks if supported + * Default implementation assumes symlinks are supported, and that toRealPath may return a valid path. + * This implementation may be overridden for NIO implementations that do not support symbolic links (For example the Azure NIO library) + */ + def getSymlinkSafePath(options: LinkOption*): Path = toRealPath(options: _*) } diff --git a/core/src/main/scala/cromwell/core/path/Obsolete.scala b/core/src/main/scala/cromwell/core/path/Obsolete.scala index 77fd32b54a7..78c346ce09d 100644 --- a/core/src/main/scala/cromwell/core/path/Obsolete.scala +++ b/core/src/main/scala/cromwell/core/path/Obsolete.scala @@ -47,16 +47,14 @@ object Obsolete { val File = ObsoleteFile object ObsoleteFile { - def newTemporaryDirectory(prefix: String = ""): DefaultPath = { + def newTemporaryDirectory(prefix: String = ""): DefaultPath = DefaultPath(better.files.File.newTemporaryDirectory(prefix).path) - } - def newTemporaryFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = { + def newTemporaryFile(prefix: String = "", suffix: String = "", parent: Option[Path] = None): Path = parent match { case Some(dir) => dir.createTempFile(prefix, suffix) case _ => DefaultPathBuilder.createTempFile(prefix, suffix) } - } def apply(path: String, fragments: String*) = DefaultPath(better.files.File(path, fragments: _*).path) diff --git a/core/src/main/scala/cromwell/core/path/PathBuilder.scala b/core/src/main/scala/cromwell/core/path/PathBuilder.scala index 371c9e98157..fecfaebd507 100644 --- a/core/src/main/scala/cromwell/core/path/PathBuilder.scala +++ b/core/src/main/scala/cromwell/core/path/PathBuilder.scala @@ -40,6 +40,7 @@ trait PreResolvePathBuilder extends PathBuilder { * @see [[cromwell.core.path.EvenBetterPathMethods]] */ trait Path extends PathObjectMethods with NioPathMethods with BetterFileMethods with EvenBetterPathMethods { + /** * A reference to the underlying nioPath, used to create new java.nio.Path's that will then be sent to newPath * for wrapping. @@ -132,11 +133,11 @@ trait Path extends PathObjectMethods with NioPathMethods with BetterFileMethods def pathWithoutScheme: String // Used by various extension traits within this scala package - private[path] final def nioPathPrivate: NioPath = nioPath + final private[path] def nioPathPrivate: NioPath = nioPath // Used within BetterFileMethods - private[path] final def betterFile: better.files.File = nioPathPrivate + final private[path] def betterFile: better.files.File = nioPathPrivate // Some Path methods return null. - private[path] final def newPathOrNull(nioPath: NioPath) = Option(nioPath).map(newPath).orNull + final private[path] def newPathOrNull(nioPath: NioPath) = Option(nioPath).map(newPath).orNull } diff --git a/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala index 1f215ba2933..cbc750f9f6f 100644 --- a/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala +++ b/core/src/main/scala/cromwell/core/path/PathBuilderFactory.scala @@ -5,23 +5,24 @@ import cromwell.core.{Dispatcher, WorkflowOptions} import cats.syntax.traverse._ import cats.instances.list._ import cats.instances.future._ +import cromwell.core.path.PathBuilderFactory.PriorityStandard import scala.concurrent.{ExecutionContext, Future} object PathBuilderFactory { // Given a list of factories, instantiates the corresponding path builders - def instantiatePathBuilders(factories: List[PathBuilderFactory], workflowOptions: WorkflowOptions)(implicit as: ActorSystem): Future[List[PathBuilder]] = { + def instantiatePathBuilders(factories: List[PathBuilderFactory], workflowOptions: WorkflowOptions)(implicit + as: ActorSystem + ): Future[List[PathBuilder]] = { implicit val ec: ExecutionContext = as.dispatchers.lookup(Dispatcher.IoDispatcher) - // The DefaultPathBuilderFactory always needs to be last. - // The reason is path builders are tried in order, and the default one is very generous in terms of paths it "thinks" it supports - // For instance, it will return a Path for a gcs url even though it doesn't really support it - val sortedFactories = factories.sortWith({ - case (_, DefaultPathBuilderFactory) => true - case (DefaultPathBuilderFactory, _) => false - case (a, b) => factories.indexOf(a) < factories.indexOf(b) - }) + val sortedFactories = factories.sortBy(_.priority) sortedFactories.traverse(_.withOptions(workflowOptions)) } + + val PriorityBlob = + 100 // High priority to evaluate first, because blob files may inadvertently match other filesystems + val PriorityStandard = 1000 + val PriorityDefault = 10000 // "Default" is a fallback, evaluate last } /** @@ -29,4 +30,11 @@ object PathBuilderFactory { */ trait PathBuilderFactory { def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext): Future[PathBuilder] + + /** + * Candidate filesystems are considered in a stable order, as some requests may match multiple filesystems. + * To customize this order, the priority of a filesystem may be adjusted. Lower number == higher priority. + * @return This filesystem's priority + */ + def priority: Int = PriorityStandard } diff --git a/core/src/main/scala/cromwell/core/path/PathCopier.scala b/core/src/main/scala/cromwell/core/path/PathCopier.scala index b9352cb2082..6848ab3d24b 100644 --- a/core/src/main/scala/cromwell/core/path/PathCopier.scala +++ b/core/src/main/scala/cromwell/core/path/PathCopier.scala @@ -18,7 +18,7 @@ object PathCopier { val tokens2 = string2.split(regexIncludingSlashes) val matchingTokens: Array[(String, String)] = tokens1.zip(tokens2).takeWhile(Function.tupled(_ == _)) - val matchingPrefix = matchingTokens.map({ case (str, _) => str }).mkString + val matchingPrefix = matchingTokens.map { case (str, _) => str }.mkString string2.stripPrefix(matchingPrefix).replaceAll("^/+", "") } @@ -39,13 +39,12 @@ object PathCopier { /** * Copies from source to destination. NOTE: Copies are not atomic, and may create a partial copy. */ - def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = { + def copy(sourceFilePath: Path, destinationFilePath: Path): Try[Unit] = Try { Option(destinationFilePath.parent).foreach(_.createDirectories()) sourceFilePath.copyTo(destinationFilePath, overwrite = true) () - } recoverWith { - case ex => Failure(new IOException(s"Failed to copy $sourceFilePath to $destinationFilePath", ex)) + } recoverWith { case ex => + Failure(new IOException(s"Failed to copy $sourceFilePath to $destinationFilePath", ex)) } - } } diff --git a/core/src/main/scala/cromwell/core/path/PathFactory.scala b/core/src/main/scala/cromwell/core/path/PathFactory.scala index a9e074afcc7..b2777b0fc83 100644 --- a/core/src/main/scala/cromwell/core/path/PathFactory.scala +++ b/core/src/main/scala/cromwell/core/path/PathFactory.scala @@ -15,6 +15,7 @@ import scala.util.{Failure, Success, Try} * Convenience trait delegating to the PathFactory singleton */ trait PathFactory { + /** * Path builders to be applied (in order) to attempt to build a Path from a string. */ @@ -43,11 +44,13 @@ object PathFactory { private def findFirstSuccess(string: String, allPathBuilders: PathBuilders, restPathBuilders: PathBuilders, - failures: Vector[String]): ErrorOr[Path] = restPathBuilders match { - case Nil => NonEmptyList.fromList(failures.toList) match { - case Some(errors) => Invalid(errors) - case None => s"Could not parse '$string' to path. No PathBuilders were provided".invalidNel - } + failures: Vector[String] + ): ErrorOr[Path] = restPathBuilders match { + case Nil => + NonEmptyList.fromList(failures.toList) match { + case Some(errors) => Invalid(errors) + case None => s"Could not parse '$string' to path. No PathBuilders were provided".invalidNel + } case pb :: rest => pb.build(string, allPathBuilders) match { case Success(path) => @@ -64,7 +67,8 @@ object PathFactory { def buildPath(string: String, pathBuilders: PathBuilders, preMapping: String => String = identity[String], - postMapping: Path => Path = identity[Path]): Path = { + postMapping: Path => Path = identity[Path] + ): Path = { lazy val pathBuilderNames: String = pathBuilders map { _.name } mkString ", " @@ -77,12 +81,12 @@ object PathFactory { path match { case Valid(v) => v case Invalid(errors) => - throw PathParsingException( - s"""Could not build the path "$string". It may refer to a filesystem not supported by this instance of Cromwell.""" + - s" Supported filesystems are: $pathBuilderNames." + - s" Failures: ${errors.toList.mkString(System.lineSeparator, System.lineSeparator, System.lineSeparator)}" + - s" Please refer to the documentation for more information on how to configure filesystems: http://cromwell.readthedocs.io/en/develop/backends/HPC/#filesystems" - ) + throw PathParsingException( + s"""Could not build the path "$string". It may refer to a filesystem not supported by this instance of Cromwell.""" + + s" Supported filesystems are: $pathBuilderNames." + + s" Failures: ${errors.toList.mkString(System.lineSeparator, System.lineSeparator, System.lineSeparator)}" + + s" Please refer to the documentation for more information on how to configure filesystems: http://cromwell.readthedocs.io/en/develop/backends/HPC/#filesystems" + ) } } } diff --git a/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala b/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala index d109faa797c..ab3d981316c 100644 --- a/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala +++ b/core/src/main/scala/cromwell/core/path/PathObjectMethods.scala @@ -8,12 +8,11 @@ trait PathObjectMethods { override def toString: String = pathAsString - override def equals(obj: Any) = { + override def equals(obj: Any) = obj match { case other: Path => nioPathPrivate == other.nioPathPrivate case _ => false } - } override def hashCode = nioPathPrivate.hashCode() } diff --git a/core/src/main/scala/cromwell/core/path/PathWriter.scala b/core/src/main/scala/cromwell/core/path/PathWriter.scala index 5a602810183..ee7717b6765 100644 --- a/core/src/main/scala/cromwell/core/path/PathWriter.scala +++ b/core/src/main/scala/cromwell/core/path/PathWriter.scala @@ -59,7 +59,7 @@ case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { * * @return a descriptive tail of the `path` and the last `tailedLines` written. */ - def tailString: String = { + def tailString: String = if (tailedLines.isEmpty) { s"Contents of $path were empty." } else if (isTailed) { @@ -67,5 +67,4 @@ case class TailedWriter(path: Path, tailedSize: Int) extends PathWriter { } else { s"Contents of $path:\n${tailedLines.mkString("\n")}" } - } } diff --git a/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala b/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala index 0a50b79e091..c901bc0d1ec 100644 --- a/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala +++ b/core/src/main/scala/cromwell/core/retry/GoogleBackoff.scala @@ -8,47 +8,59 @@ import net.ceedubs.ficus.Ficus._ import scala.concurrent.duration.{Duration, FiniteDuration} object InitialGapBackoff { - def apply(initialGap: FiniteDuration, initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double) = { - new InitialGapBackoff(initialGap, new ExponentialBackOff.Builder() - .setInitialIntervalMillis(initialInterval.toMillis.toInt) - .setMaxIntervalMillis(maxInterval.toMillis.toInt) - .setMultiplier(multiplier) - .setMaxElapsedTimeMillis(Int.MaxValue) - .build()) - } + def apply(initialGap: FiniteDuration, + initialInterval: FiniteDuration, + maxInterval: FiniteDuration, + multiplier: Double + ) = + new InitialGapBackoff( + initialGap, + new ExponentialBackOff.Builder() + .setInitialIntervalMillis(initialInterval.toMillis.toInt) + .setMaxIntervalMillis(maxInterval.toMillis.toInt) + .setMultiplier(multiplier) + .setMaxElapsedTimeMillis(Int.MaxValue) + .build() + ) } case class InitialGapBackoff(initialGapMillis: FiniteDuration, googleBackoff: ExponentialBackOff) extends Backoff { assert(initialGapMillis.compareTo(Duration.Zero) != 0, "Initial gap cannot be null, use SimpleBackoff instead.") override val backoffMillis = initialGapMillis.toMillis + /** Switch to a SimpleExponentialBackoff after the initial gap has been used */ override def next = new SimpleExponentialBackoff(googleBackoff) } object SimpleExponentialBackoff { - def apply(initialInterval: FiniteDuration, maxInterval: FiniteDuration, multiplier: Double, randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR) = { - new SimpleExponentialBackoff(new ExponentialBackOff.Builder() - .setInitialIntervalMillis(initialInterval.toMillis.toInt) - .setMaxIntervalMillis(maxInterval.toMillis.toInt) - .setMultiplier(multiplier) - .setMaxElapsedTimeMillis(Int.MaxValue) - .setRandomizationFactor(randomizationFactor) - .build()) - } - - def apply(config: Config): SimpleExponentialBackoff = { + def apply(initialInterval: FiniteDuration, + maxInterval: FiniteDuration, + multiplier: Double, + randomizationFactor: Double = ExponentialBackOff.DEFAULT_RANDOMIZATION_FACTOR + ) = + new SimpleExponentialBackoff( + new ExponentialBackOff.Builder() + .setInitialIntervalMillis(initialInterval.toMillis.toInt) + .setMaxIntervalMillis(maxInterval.toMillis.toInt) + .setMultiplier(multiplier) + .setMaxElapsedTimeMillis(Int.MaxValue) + .setRandomizationFactor(randomizationFactor) + .build() + ) + + def apply(config: Config): SimpleExponentialBackoff = SimpleExponentialBackoff( config.as[FiniteDuration]("min"), config.as[FiniteDuration]("max"), config.as[Double]("multiplier"), config.as[Double]("randomization-factor") ) - } } case class SimpleExponentialBackoff(googleBackoff: ExponentialBackOff) extends Backoff { override def backoffMillis = googleBackoff.nextBackOffMillis() + /** google ExponentialBackOff is mutable so we can keep returning the same instance */ override def next = this } diff --git a/core/src/main/scala/cromwell/core/retry/Retry.scala b/core/src/main/scala/cromwell/core/retry/Retry.scala index 5e5ba4fe6b1..a7c10cd48dd 100644 --- a/core/src/main/scala/cromwell/core/retry/Retry.scala +++ b/core/src/main/scala/cromwell/core/retry/Retry.scala @@ -34,11 +34,11 @@ object Retry extends StrictLogging { */ def withRetry[A](f: () => Future[A], maxRetries: Option[Int] = Option(10), - backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D), + backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1d), isTransient: Throwable => Boolean = throwableToFalse, isFatal: Throwable => Boolean = throwableToFalse, - onRetry: Throwable => Unit = noopOnRetry) - (implicit actorSystem: ActorSystem): Future[A] = { + onRetry: Throwable => Unit = noopOnRetry + )(implicit actorSystem: ActorSystem): Future[A] = { // In the future we might want EC passed in separately but at the moment it caused more issues than it solved to do so implicit val ec: ExecutionContext = actorSystem.dispatcher val delay = backoff.backoffMillis.millis @@ -47,10 +47,18 @@ object Retry extends StrictLogging { case throwable if isFatal(throwable) => Future.failed(CromwellFatalException(throwable)) case throwable if !isFatal(throwable) => val retriesLeft = if (isTransient(throwable)) maxRetries else maxRetries map { _ - 1 } - + if (retriesLeft.forall(_ > 0)) { onRetry(throwable) - after(delay, actorSystem.scheduler)(withRetry(f, backoff = backoff.next, maxRetries = retriesLeft, isTransient = isTransient, isFatal = isFatal, onRetry = onRetry)) + after(delay, actorSystem.scheduler)( + withRetry(f, + backoff = backoff.next, + maxRetries = retriesLeft, + isTransient = isTransient, + isFatal = isFatal, + onRetry = onRetry + ) + ) } else { Future.failed(new CromwellFatalException(throwable)) } @@ -69,8 +77,8 @@ object Retry extends StrictLogging { */ def withRetryForTransactionRollback[A](f: () => Future[A], maxRetries: Int = 5, - backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1D)) - (implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[A] = { + backoff: Backoff = SimpleExponentialBackoff(5 seconds, 10 seconds, 1.1d) + )(implicit actorSystem: ActorSystem, ec: ExecutionContext): Future[A] = { val delay = backoff.backoffMillis.millis f() recoverWith { @@ -85,4 +93,3 @@ object Retry extends StrictLogging { } } } - diff --git a/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala b/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala index bfae7232d64..d2387b5062a 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WomValueBuilder.scala @@ -8,7 +8,6 @@ import wom.values._ import scala.language.postfixOps - /** * Builds arbitrary `WomValues` from `WomValueSimpletons`. **/ @@ -63,28 +62,26 @@ object WomValueBuilder { private val MapElementPattern = raw"^:((?:\\[]\[:]|[^]\[:])+)(.*)".r // Group tuples by key using a Map with key type `K`. - private def group[K](tuples: Iterable[(K, SimpletonComponent)]): Map[K, Iterable[SimpletonComponent]] = { - tuples groupBy { case (i, _) => i } map { case (k, v) => k -> (v map { case (_, s) => s}) } - } + private def group[K](tuples: Iterable[(K, SimpletonComponent)]): Map[K, Iterable[SimpletonComponent]] = + tuples groupBy { case (i, _) => i } map { case (k, v) => k -> (v map { case (_, s) => s }) } // Returns a tuple of the index into the outermost array and a `SimpletonComponent` whose path reflects the "descent" // into the array. e.g. for a component // SimpletonComponent("[0][1]", v) this would return (0 -> SimpletonComponent("[1]", v)). - private def descendIntoArray(component: SimpletonComponent): (Int, SimpletonComponent) = { - component.path match { case ArrayElementPattern(index, more) => index.toInt -> component.copy(path = more)} - } + private def descendIntoArray(component: SimpletonComponent): (Int, SimpletonComponent) = + component.path match { case ArrayElementPattern(index, more) => index.toInt -> component.copy(path = more) } // Returns a tuple of the key into the outermost map and a `SimpletonComponent` whose path reflects the "descent" // into the map. e.g. for a component // SimpletonComponent(":bar:baz", v) this would return ("bar" -> SimpletonComponent(":baz", v)). // Map keys are treated as Strings by this method, the caller must ultimately do the appropriate coercion to the // actual map key type. - private def descendIntoMap(component: SimpletonComponent): (String, SimpletonComponent) = { - component.path match { case MapElementPattern(key, more) => key.unescapeMeta -> component.copy(path = more)} - } + private def descendIntoMap(component: SimpletonComponent): (String, SimpletonComponent) = + component.path match { case MapElementPattern(key, more) => key.unescapeMeta -> component.copy(path = more) } - private implicit class EnhancedSimpletonComponents(val components: Iterable[SimpletonComponent]) extends AnyVal { - def asArray: List[Iterable[SimpletonComponent]] = group(components map descendIntoArray).toList.sortBy(_._1).map(_._2) + implicit private class EnhancedSimpletonComponents(val components: Iterable[SimpletonComponent]) extends AnyVal { + def asArray: List[Iterable[SimpletonComponent]] = + group(components map descendIntoArray).toList.sortBy(_._1).map(_._2) def asMap: Map[String, Iterable[SimpletonComponent]] = group(components map descendIntoMap) def asPrimitive: WomValue = components.head.value def asString: String = asPrimitive.valueString @@ -92,51 +89,50 @@ object WomValueBuilder { private def toWomValue(outputType: WomType, components: Iterable[SimpletonComponent]): WomValue = { - - // Returns a tuple of the key into the pair (i.e. left or right) and a `SimpletonComponent` whose path reflects the "descent" // into the pair. e.g. for a component // SimpletonComponent(":left:foo", someValue) this would return (PairLeft -> SimpletonComponent(":baz", someValue)). sealed trait PairLeftOrRight case object PairLeft extends PairLeftOrRight case object PairRight extends PairLeftOrRight - def descendIntoPair(component: SimpletonComponent): (PairLeftOrRight, SimpletonComponent) = { + def descendIntoPair(component: SimpletonComponent): (PairLeftOrRight, SimpletonComponent) = component.path match { case MapElementPattern("left", more) => PairLeft -> component.copy(path = more) case MapElementPattern("right", more) => PairRight -> component.copy(path = more) } - } - def toWomFile(components: Iterable[SimpletonComponent]) = { + def toWomFile(components: Iterable[SimpletonComponent]) = // If there's just one simpleton, it's a primitive (file or directory) if (components.size == 1) components.asPrimitive else { // Otherwise make a map of the components and detect the type of file from the class field val groupedListing = components.asMap - def isClass(className: String) = { - groupedListing.get(ClassKey) - /* If the class field is in an array it will be prefixed with a ':', so check for that as well. - * e.g: secondaryFiles[0]:class -> "File" - * secondaryFiles[0]:value -> "file/path" - * would produce a Map( - * ":class" -> List(Simpleton("File")), - * ":value" -> List(Simpleton("file/path")) - * ) - */ - .orElse(groupedListing.get(s":$ClassKey")) - .map(_.asPrimitive.valueString) - .contains(className) - } + def isClass(className: String) = + groupedListing + .get(ClassKey) + /* If the class field is in an array it will be prefixed with a ':', so check for that as well. + * e.g: secondaryFiles[0]:class -> "File" + * secondaryFiles[0]:value -> "file/path" + * would produce a Map( + * ":class" -> List(Simpleton("File")), + * ":value" -> List(Simpleton("file/path")) + * ) + */ + .orElse(groupedListing.get(s":$ClassKey")) + .map(_.asPrimitive.valueString) + .contains(className) def isDirectory = isClass(WomValueSimpleton.DirectoryClass) def isFile = isClass(WomValueSimpleton.FileClass) if (isDirectory) toWomValue(WomMaybeListedDirectoryType, components) else if (isFile) toWomValue(WomMaybePopulatedFileType, components) - else throw new IllegalArgumentException(s"There is no WomFile that can be built from simpletons: ${groupedListing.toList.mkString(", ")}") + else + throw new IllegalArgumentException( + s"There is no WomFile that can be built from simpletons: ${groupedListing.toList.mkString(", ")}" + ) } - } outputType match { case _: WomPrimitiveType => @@ -151,19 +147,31 @@ object WomValueBuilder { WomArray(arrayType, components.asArray map { toWomValue(arrayType.memberType, _) }) case mapType: WomMapType => // map keys are guaranteed by WOM to be primitives, so the "coerceRawValue(..).get" is safe. - WomMap(mapType, components.asMap map { case (k, ss) => mapType.keyType.coerceRawValue(k).get -> toWomValue(mapType.valueType, ss) }) + WomMap(mapType, + components.asMap map { case (k, ss) => + mapType.keyType.coerceRawValue(k).get -> toWomValue(mapType.valueType, ss) + } + ) case pairType: WomPairType => - val groupedByLeftOrRight: Map[PairLeftOrRight, Iterable[SimpletonComponent]] = group(components map descendIntoPair) - WomPair(toWomValue(pairType.leftType, groupedByLeftOrRight(PairLeft)), toWomValue(pairType.rightType, groupedByLeftOrRight(PairRight))) + val groupedByLeftOrRight: Map[PairLeftOrRight, Iterable[SimpletonComponent]] = group( + components map descendIntoPair + ) + WomPair(toWomValue(pairType.leftType, groupedByLeftOrRight(PairLeft)), + toWomValue(pairType.rightType, groupedByLeftOrRight(PairRight)) + ) case WomObjectType => // map keys are guaranteed by WOM to be primitives, so the "coerceRawValue(..).get" is safe. val map: Map[String, WomValue] = components.asMap map { case (k, ss) => k -> toWomValue(WomAnyType, ss) } WomObject(map) case composite: WomCompositeType => val map: Map[String, WomValue] = components.asMap map { case (k, ss) => - val valueType = composite - .typeMap - .getOrElse(k, throw new RuntimeException(s"Field $k is not a declared field of composite type $composite. Cannot build a WomValue from the simpletons.")) + val valueType = composite.typeMap + .getOrElse( + k, + throw new RuntimeException( + s"Field $k is not a declared field of composite type $composite. Cannot build a WomValue from the simpletons." + ) + ) k -> toWomValue(valueType, ss) } WomObject.withTypeUnsafe(map, composite) @@ -171,8 +179,9 @@ object WomValueBuilder { val directoryValues = components.asMap val value = directoryValues.get("value").map(_.asString) - val listing = directoryValues.get("listing") - .map({ _.asArray.map(toWomFile).collect({ case womFile: WomFile => womFile }) }) + val listing = directoryValues + .get("listing") + .map(_.asArray.map(toWomFile).collect { case womFile: WomFile => womFile }) WomMaybeListedDirectory(value, listing) case WomMaybePopulatedFileType => @@ -183,9 +192,9 @@ object WomValueBuilder { val size = populatedValues.get("size").map(_.asString.toLong) val format = populatedValues.get("format").map(_.asString) val contents = populatedValues.get("contents").map(_.asString) - val secondaryFiles = populatedValues.get("secondaryFiles").toList.flatMap({ - _.asArray.map(toWomFile).collect({ case womFile: WomFile => womFile }) - }) + val secondaryFiles = populatedValues.get("secondaryFiles").toList.flatMap { + _.asArray.map(toWomFile).collect { case womFile: WomFile => womFile } + } WomMaybePopulatedFile( valueOption = value, @@ -234,23 +243,28 @@ object WomValueBuilder { */ private case class SimpletonComponent(path: String, value: WomValue) - def toJobOutputs(taskOutputs: Iterable[OutputPort], simpletons: Iterable[WomValueSimpleton]): CallOutputs = { + def toJobOutputs(taskOutputs: Iterable[OutputPort], simpletons: Iterable[WomValueSimpleton]): CallOutputs = CallOutputs(toWomValues(taskOutputs, simpletons)) - } - def toWomValues(taskOutputs: Iterable[OutputPort], simpletons: Iterable[WomValueSimpleton]): Map[OutputPort, WomValue] = { + def toWomValues(taskOutputs: Iterable[OutputPort], + simpletons: Iterable[WomValueSimpleton] + ): Map[OutputPort, WomValue] = { - def simpletonToComponent(name: String)(simpleton: WomValueSimpleton): SimpletonComponent = { + def simpletonToComponent(name: String)(simpleton: WomValueSimpleton): SimpletonComponent = SimpletonComponent(simpleton.simpletonKey.drop(name.length), simpleton.simpletonValue) - } // This is meant to "rehydrate" simpletonized WomValues back to WomValues. It is assumed that these WomValues were // "dehydrated" to WomValueSimpletons correctly. This code is not robust to corrupt input whatsoever. val types = taskOutputs map { o => o -> o.womType } toMap - val simpletonsByOutputName = simpletons groupBy { _.simpletonKey match { case IdentifierAndPathPattern(i, _) => i } } + val simpletonsByOutputName = simpletons groupBy { + _.simpletonKey match { case IdentifierAndPathPattern(i, _) => i } + } val simpletonComponentsByOutputName: Map[String, Iterable[SimpletonComponent]] = simpletonsByOutputName map { case (name, ss) => name -> (ss map simpletonToComponent(name)) } - types map { case (outputPort, outputType) => outputPort -> toWomValue(outputType, simpletonComponentsByOutputName.getOrElse(outputPort.internalName, Seq.empty))} + types map { case (outputPort, outputType) => + outputPort -> toWomValue(outputType, + simpletonComponentsByOutputName.getOrElse(outputPort.internalName, Seq.empty) + ) + } } } - diff --git a/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala b/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala index 01ec7ddf8c8..35e7accdf07 100644 --- a/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala +++ b/core/src/main/scala/cromwell/core/simpleton/WomValueSimpleton.scala @@ -15,7 +15,7 @@ case class WomValueSimpleton(simpletonKey: String, simpletonValue: WomPrimitive) * `WomValueSimpleton`s are transformed back to `WomValue`s. */ object WomValueSimpleton { - + val ClassKey = "class" val DirectoryClass = "Directory" val FileClass = "File" @@ -35,23 +35,30 @@ object WomValueSimpleton { private def toNumberSimpleton(key: String)(value: Long) = WomValueSimpleton(key, WomInteger(value.toInt)) // Pass the simplifyMode down to recursive calls without having to sling the parameter around explicitly. - def simplify(name: String)(implicit simplifyMode: SimplifyMode = SimplifyMode(forCaching = false)): Iterable[WomValueSimpleton] = { + def simplify( + name: String + )(implicit simplifyMode: SimplifyMode = SimplifyMode(forCaching = false)): Iterable[WomValueSimpleton] = { def suffix(suffix: String) = s"$name:$suffix" val fileValueSimplifier: String => String => WomValueSimpleton = if (simplifyMode.forCaching) key => value => WomValueSimpleton(key, WomSingleFile(value)) else toStringSimpleton // What should this even do? Maybe just pick out the last bit of the path and store that as a String? val directoryValueSimplifier: String => String => WomValueSimpleton = - if (simplifyMode.forCaching) key => value => WomValueSimpleton(key, WomString(value.substring(value.lastIndexOf("/") + 1))) else toStringSimpleton + if (simplifyMode.forCaching) + key => value => WomValueSimpleton(key, WomString(value.substring(value.lastIndexOf("/") + 1))) + else toStringSimpleton womValue match { case prim: WomPrimitive => List(WomValueSimpleton(name, prim)) case opt: WomOptionalValue => opt.value.map(_.simplify(name)).getOrElse(Seq.empty) - case WomArray(_, arrayValue) => arrayValue.zipWithIndex flatMap { case (arrayItem, index) => arrayItem.simplify(s"$name[$index]") } - case WomMap(_, mapValue) => mapValue flatMap { case (key, value) => value.simplify(s"$name:${key.valueString.escapeMeta}") } + case WomArray(_, arrayValue) => + arrayValue.zipWithIndex flatMap { case (arrayItem, index) => arrayItem.simplify(s"$name[$index]") } + case WomMap(_, mapValue) => + mapValue flatMap { case (key, value) => value.simplify(s"$name:${key.valueString.escapeMeta}") } case WomPair(left, right) => left.simplify(s"$name:left") ++ right.simplify(s"$name:right") - case womObjectLike: WomObjectLike => womObjectLike.values flatMap { - case (key, value) => value.simplify(s"$name:${key.escapeMeta}") - } + case womObjectLike: WomObjectLike => + womObjectLike.values flatMap { case (key, value) => + value.simplify(s"$name:${key.escapeMeta}") + } case WomMaybeListedDirectory(valueOption, listingOption, _, _) => // This simpleton is not strictly part of the WomFile but is used to record the type of this WomValue so it can // be re-built appropriately in the WomValueBuilder @@ -82,10 +89,14 @@ object WomValueSimpleton { } implicit class WomValuesSimplifier(womValues: Map[String, WomValue]) { - def simplifyForCaching: Iterable[WomValueSimpleton] = womValues flatMap { case (name, value) => value.simplify(name)(simplifyMode = SimplifyMode(forCaching = true)) } + def simplifyForCaching: Iterable[WomValueSimpleton] = womValues flatMap { case (name, value) => + value.simplify(name)(simplifyMode = SimplifyMode(forCaching = true)) + } } implicit class WomValuesSimplifierPort(womValues: Map[OutputPort, WomValue]) { - def simplify: Iterable[WomValueSimpleton] = womValues flatMap { case (port, value) => value.simplify(port.internalName) } + def simplify: Iterable[WomValueSimpleton] = womValues flatMap { case (port, value) => + value.simplify(port.internalName) + } } } diff --git a/core/src/main/scala/cromwell/util/DatabaseUtil.scala b/core/src/main/scala/cromwell/util/DatabaseUtil.scala index d5033415df1..af1b7c176d2 100644 --- a/core/src/main/scala/cromwell/util/DatabaseUtil.scala +++ b/core/src/main/scala/cromwell/util/DatabaseUtil.scala @@ -16,7 +16,7 @@ object DatabaseUtil { } def withRetry[A](f: () => Future[A])(implicit actorSystem: ActorSystem): Future[A] = { - val RetryBackoff = SimpleExponentialBackoff(50 millis, 1 seconds, 1D) + val RetryBackoff = SimpleExponentialBackoff(50 millis, 1 seconds, 1d) Retry.withRetry(f, maxRetries = Option(10), backoff = RetryBackoff, isTransient = isTransient) } } diff --git a/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala b/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala index 5ed66fb5e9c..5a1ea5ef2d9 100644 --- a/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala +++ b/core/src/main/scala/cromwell/util/GracefulShutdownHelper.scala @@ -12,10 +12,10 @@ object GracefulShutdownHelper { trait GracefulShutdownHelper extends GracefulStopSupport { this: Actor with ActorLogging => private var shuttingDown: Boolean = false private var shutdownList: Set[ActorRef] = Set.empty - + def isShuttingDown: Boolean = shuttingDown - def waitForActorsAndShutdown(actorsLists: NonEmptyList[ActorRef]): Unit = { + def waitForActorsAndShutdown(actorsLists: NonEmptyList[ActorRef]): Unit = if (shuttingDown) { log.error("Programmer error, this actor has already initiated its shutdown. Only call this once per actor !") } else { @@ -23,12 +23,11 @@ trait GracefulShutdownHelper extends GracefulStopSupport { this: Actor with Acto shutdownList = actorsLists.toList.toSet shutdownList foreach context.watch shutdownList foreach { _ ! ShutdownCommand } - + context become { case Terminated(actor) if shuttingDown && shutdownList.contains(actor) => shutdownList = shutdownList - actor if (shutdownList.isEmpty) context stop self } } - } } diff --git a/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala b/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala index 811f9dce61f..1de0da45989 100644 --- a/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala +++ b/core/src/main/scala/cromwell/util/JsonFormatting/WomValueJsonFormatter.scala @@ -12,15 +12,15 @@ object WomValueJsonFormatter extends DefaultJsonProtocol { case f: WomFloat => JsNumber(f.value) case b: WomBoolean => JsBoolean(b.value) case f: WomSingleFile => JsString(f.value) - case o: WomObjectLike => new JsObject(o.values map {case(k, v) => k -> write(v)}) + case o: WomObjectLike => new JsObject(o.values map { case (k, v) => k -> write(v) }) case a: WomArray => new JsArray(a.value.map(write).toVector) - case m: WomMap => new JsObject(m.value map {case(k,v) => k.valueString -> write(v)}) + case m: WomMap => new JsObject(m.value map { case (k, v) => k.valueString -> write(v) }) case q: WomPair => new JsObject(Map("left" -> write(q.left), "right" -> write(q.right))) case WomOptionalValue(_, Some(innerValue)) => write(innerValue) case WomOptionalValue(_, None) => JsNull case WomCoproductValue(_, innerValue) => write(innerValue) case WomEnumerationValue(_, innerValue) => JsString(innerValue) - // handles WdlExpression + // handles WdlExpression case v: WomValue => JsString(v.toWomString) } @@ -31,7 +31,7 @@ object WomValueJsonFormatter extends DefaultJsonProtocol { // In addition, we make a lot of assumptions about what type of WomValue to create. Oh well... it should all fall out in the coercion (fingercrossed)! def read(value: JsValue): WomValue = value match { case JsObject(fields) => - val wdlFields: Map[WomValue, WomValue] = fields map {case (k, v) => WomString(k) -> read(v)} + val wdlFields: Map[WomValue, WomValue] = fields map { case (k, v) => WomString(k) -> read(v) } if (fields.isEmpty) WomMap(WomMapType(WomStringType, WomStringType), Map.empty[WomValue, WomValue]) else WomMap(WomMapType(wdlFields.head._1.womType, wdlFields.head._2.womType), wdlFields) case JsArray(vector) if vector.nonEmpty => WomArray(WomArrayType(read(vector.head).womType), vector map read) @@ -53,4 +53,3 @@ object WomSingleFileJsonFormatter extends DefaultJsonProtocol { } } } - diff --git a/core/src/main/scala/cromwell/util/PromiseActor.scala b/core/src/main/scala/cromwell/util/PromiseActor.scala index bd5efa5b0c4..efe3f0cb217 100644 --- a/core/src/main/scala/cromwell/util/PromiseActor.scala +++ b/core/src/main/scala/cromwell/util/PromiseActor.scala @@ -17,7 +17,10 @@ private class PromiseActor(promise: Promise[Any], sendTo: ActorRef, msg: Any) ex if (actorRef == sendTo) { promise.tryFailure(new RuntimeException("Promise-watched actor completed before sending back a message")) } else { - log.error("Spooky happenstances! A Terminated({}) message was sent to a private Promise actor which wasn't watching it!?", actorRef) + log.error( + "Spooky happenstances! A Terminated({}) message was sent to a private Promise actor which wasn't watching it!?", + actorRef + ) } context.stop(self) case success => @@ -27,6 +30,7 @@ private class PromiseActor(promise: Promise[Any], sendTo: ActorRef, msg: Any) ex } object PromiseActor { + /** * Sends a message to an actor and returns the future associated with the fullfilment of the reply * Can be used instead of the akka `ask` semantics, without any timeout @@ -42,11 +46,11 @@ object PromiseActor { promise.future } - def props(promise: Promise[Any], sendTo: ActorRef, msg: Any): Props = Props(new PromiseActor(promise, sendTo, msg)).withDispatcher(EngineDispatcher) + def props(promise: Promise[Any], sendTo: ActorRef, msg: Any): Props = + Props(new PromiseActor(promise, sendTo, msg)).withDispatcher(EngineDispatcher) implicit class EnhancedActorRef(val actorRef: ActorRef) extends AnyVal { - def askNoTimeout(message: Any)(implicit actorRefFactory: ActorRefFactory): Future[Any] = { + def askNoTimeout(message: Any)(implicit actorRefFactory: ActorRefFactory): Future[Any] = PromiseActor.askNoTimeout(message, actorRef) - } } } diff --git a/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala index 238cdd70573..50a43cf2987 100644 --- a/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala +++ b/core/src/main/scala/cromwell/util/StopAndLogSupervisor.scala @@ -8,13 +8,12 @@ trait StopAndLogSupervisor { this: Actor => protected def onFailure(actorRef: ActorRef, throwable: => Throwable): Unit final val stopAndLogStrategy: SupervisorStrategy = { - def stoppingDecider: Decider = { - case e: Exception => - onFailure(sender(), e) - Stop + def stoppingDecider: Decider = { case e: Exception => + onFailure(sender(), e) + Stop } OneForOneStrategy(loggingEnabled = false)(stoppingDecider) } - override final val supervisorStrategy = stopAndLogStrategy + final override val supervisorStrategy = stopAndLogStrategy } diff --git a/core/src/main/scala/cromwell/util/TryWithResource.scala b/core/src/main/scala/cromwell/util/TryWithResource.scala index fdbfbda1882..6909232150d 100644 --- a/core/src/main/scala/cromwell/util/TryWithResource.scala +++ b/core/src/main/scala/cromwell/util/TryWithResource.scala @@ -20,17 +20,17 @@ object TryWithResource { case x: Throwable => t = Option(x) throw x - } finally { + } finally resource foreach { r => - try { + try r.close() - } catch { - case y: Throwable => t match { - case Some(_t) => _t.addSuppressed(y) - case None => throw y - } + catch { + case y: Throwable => + t match { + case Some(_t) => _t.addSuppressed(y) + case None => throw y + } } } - } } } diff --git a/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala b/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala index 25418a88fa3..4a3dc8c4929 100644 --- a/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala +++ b/core/src/test/scala/cromwell/core/DockerCredentialsSpec.scala @@ -18,11 +18,11 @@ class DockerCredentialsSpec extends AnyFlatSpec with Matchers { val credentials: Any = new DockerCredentials(Base64.getEncoder.encodeToString(tokenString.getBytes()), None, None) credentials match { - case DockerCredentialUsernameAndPassword(u, p) => { + case DockerCredentialUsernameAndPassword(u, p) => u should be(expectedUsername) p should be(expectedPassword) - } - case _ => fail(s"Expected to decompose ${tokenString} into username=$expectedPassword and password=$expectedPassword") + case _ => + fail(s"Expected to decompose ${tokenString} into username=$expectedPassword and password=$expectedPassword") } } } diff --git a/core/src/test/scala/cromwell/core/LoadConfigSpec.scala b/core/src/test/scala/cromwell/core/LoadConfigSpec.scala index 0ac4947aa1c..b2008ef3aa0 100644 --- a/core/src/test/scala/cromwell/core/LoadConfigSpec.scala +++ b/core/src/test/scala/cromwell/core/LoadConfigSpec.scala @@ -8,7 +8,7 @@ import scala.concurrent.duration._ class LoadConfigSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "LoadConfig" - + it should "parse load config" in { LoadConfig.JobStoreReadThreshold shouldBe 10000 LoadConfig.JobStoreWriteThreshold shouldBe 10000 diff --git a/core/src/test/scala/cromwell/core/MockIoActor.scala b/core/src/test/scala/cromwell/core/MockIoActor.scala index 3c1831ae8d3..dd2e8d114ac 100644 --- a/core/src/test/scala/cromwell/core/MockIoActor.scala +++ b/core/src/test/scala/cromwell/core/MockIoActor.scala @@ -18,13 +18,14 @@ class MockIoActor(returnCode: String, stderrSize: Long) extends Actor { case command: IoSizeCommand => sender() ! IoSuccess(command, 0L) case command: IoContentAsStringCommand => sender() ! IoSuccess(command, "0") case command: IoExistsCommand => sender() ! IoSuccess(command, false) - - // With context + + // With context case (requestContext: Any, command: IoCopyCommand) => sender() ! (requestContext -> IoSuccess(command, ())) case (requestContext: Any, command: IoWriteCommand) => sender() ! (requestContext -> IoSuccess(command, ())) case (requestContext: Any, command: IoDeleteCommand) => sender() ! (requestContext -> IoSuccess(command, ())) case (requestContext: Any, command: IoSizeCommand) => sender() ! (requestContext -> IoSuccess(command, stderrSize)) - case (requestContext: Any, command: IoContentAsStringCommand) => sender() ! (requestContext -> IoSuccess(command, returnCode)) + case (requestContext: Any, command: IoContentAsStringCommand) => + sender() ! (requestContext -> IoSuccess(command, returnCode)) case (requestContext: Any, command: IoExistsCommand) => sender() ! (requestContext -> IoSuccess(command, false)) case withPromise: IoCommandWithPromise[_] => self ! ((withPromise.promise, withPromise.ioCommand)) diff --git a/core/src/test/scala/cromwell/core/SimpleIoActor.scala b/core/src/test/scala/cromwell/core/SimpleIoActor.scala index 589065758c5..4ca3d3c1bdf 100644 --- a/core/src/test/scala/cromwell/core/SimpleIoActor.scala +++ b/core/src/test/scala/cromwell/core/SimpleIoActor.scala @@ -14,49 +14,44 @@ object SimpleIoActor { } class SimpleIoActor extends Actor { - + override def receive: Receive = { case command: IoCopyCommand => - Try(command.source.copyTo(command.destination)) match { case Success(_) => sender() ! IoSuccess(command, ()) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoWriteCommand => - Try(command.file.write(command.content)(command.openOptions, StandardCharsets.UTF_8)) match { case Success(_) => sender() ! IoSuccess(command, ()) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoDeleteCommand => - Try(command.file.delete(command.swallowIOExceptions)) match { case Success(_) => sender() ! IoSuccess(command, ()) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoSizeCommand => - Try(command.file.size) match { case Success(size) => sender() ! IoSuccess(command, size) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoContentAsStringCommand => - Try(command.file.contentAsString) match { case Success(content) => sender() ! IoSuccess(command, content) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoHashCommand => Try(command.file.md5) match { case Success(hash) => sender() ! IoSuccess(command, hash) case Failure(failure) => sender() ! IoFailure(command, failure) } - + case command: IoExistsCommand => Try(command.file.exists) match { case Success(exists) => sender() ! IoSuccess(command, exists) @@ -65,49 +60,42 @@ class SimpleIoActor extends Actor { // With context case (requestContext: Any, command: IoCopyCommand) => - Try(command.source.copyTo(command.destination, overwrite = true)) match { case Success(_) => sender() ! (requestContext -> IoSuccess(command, ())) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - - case (requestContext: Any, command: IoWriteCommand) => + case (requestContext: Any, command: IoWriteCommand) => Try(command.file.write(command.content)) match { case Success(_) => sender() ! (requestContext -> IoSuccess(command, ())) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - - case (requestContext: Any, command: IoDeleteCommand) => + case (requestContext: Any, command: IoDeleteCommand) => Try(command.file.delete(command.swallowIOExceptions)) match { case Success(_) => sender() ! (requestContext -> IoSuccess(command, ())) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - + case (requestContext: Any, command: IoSizeCommand) => - Try(command.file.size) match { case Success(size) => sender() ! (requestContext -> IoSuccess(command, size)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - - case (requestContext: Any, command: IoContentAsStringCommand) => + case (requestContext: Any, command: IoContentAsStringCommand) => Try(command.file.contentAsString) match { case Success(content) => sender() ! (requestContext -> IoSuccess(command, content)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } - + case (requestContext: Any, command: IoHashCommand) => - Try(command.file.md5) match { case Success(hash) => sender() ! (requestContext -> IoSuccess(command, hash)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) } case (requestContext: Any, command: IoExistsCommand) => - Try(command.file.exists) match { case Success(exists) => sender() ! (requestContext -> IoSuccess(command, exists)) case Failure(failure) => sender() ! (requestContext -> IoFailure(command, failure)) diff --git a/core/src/test/scala/cromwell/core/TestKitSuite.scala b/core/src/test/scala/cromwell/core/TestKitSuite.scala index d9fe5c3d56c..8febaa3edd8 100644 --- a/core/src/test/scala/cromwell/core/TestKitSuite.scala +++ b/core/src/test/scala/cromwell/core/TestKitSuite.scala @@ -18,9 +18,8 @@ abstract class TestKitSuite extends TestKitBase with Suite with BeforeAndAfterAl implicit lazy val system: ActorSystem = ActorSystem(actorSystemName, actorSystemConfig) - override protected def afterAll(): Unit = { + override protected def afterAll(): Unit = shutdown() - } // 'BlackHoleActor' swallows messages without logging them (thus reduces log file overhead): val emptyActor: ActorRef = system.actorOf(TestActors.blackholeProps, "TestKitSuiteEmptyActor") diff --git a/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala b/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala index 22aa4577c63..dd4de0a4773 100644 --- a/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala +++ b/core/src/test/scala/cromwell/core/WorkflowOptionsSpec.scala @@ -21,11 +21,11 @@ class WorkflowOptionsSpec extends Matchers with AnyWordSpecLike { WorkflowOptions.fromJsonObject(workflowOptionsJson) match { case Success(options) => options.get("key") shouldEqual Success("value") - options.get("bad_key") shouldBe a [Failure[_]] + options.get("bad_key") shouldBe a[Failure[_]] options.clearEncryptedValues.asPrettyJson shouldEqual """{ - | "key": "value" - |}""".stripMargin + | "key": "value" + |}""".stripMargin case _ => fail("Expecting workflow options to be parseable") } } diff --git a/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala b/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala index e426a349448..1fe0ca6658b 100644 --- a/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala +++ b/core/src/test/scala/cromwell/core/actor/BatchActorSpec.scala @@ -189,24 +189,23 @@ class BatchActorSpec extends TestKitSuite with AnyFlatSpecLike with Matchers wit } } - class BatchActorTest(processingTime: FiniteDuration = Duration.Zero, fail: Boolean = false) extends BatchActor[String](10.hours, 10) { + class BatchActorTest(processingTime: FiniteDuration = Duration.Zero, fail: Boolean = false) + extends BatchActor[String](10.hours, 10) { var processed: Vector[String] = Vector.empty - override def commandToData(snd: ActorRef) = { - case command: String => command + override def commandToData(snd: ActorRef) = { case command: String => + command } override protected def weightFunction(command: String) = command.length - override protected def process(data: NonEmptyVector[String]) = { + override protected def process(data: NonEmptyVector[String]) = if (processingTime != Duration.Zero) { processed = processed ++ data.toVector - val promise = Promise[Int]() - system.scheduler.scheduleOnce(processingTime) { promise.success(data.map(weightFunction).toVector.sum) } + val promise = Promise[Int]() + system.scheduler.scheduleOnce(processingTime)(promise.success(data.map(weightFunction).toVector.sum)) promise.future } else if (!fail) { processed = processed ++ data.toVector Future.successful(data.map(weightFunction).toVector.sum) - } - else Future.failed(new Exception("Oh nose ! (This is a test failure and is expected !)") with NoStackTrace) - } + } else Future.failed(new Exception("Oh nose ! (This is a test failure and is expected !)") with NoStackTrace) } } diff --git a/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala b/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala index 939a1aa1834..dc29807d061 100644 --- a/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala +++ b/core/src/test/scala/cromwell/core/actor/RobustClientHelperSpec.scala @@ -14,39 +14,39 @@ import scala.language.postfixOps class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender { behavior of "RobustClientHelper" - + it should "handle Backpressure responses" in { val remoteActor = TestProbe() val delegateActor = TestProbe() - + val margin = 2.second - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 10 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) - + val messageToSend = TestActor.TestMessage("hello") - - //send message + + // send message testActor.underlyingActor.sendMessage(messageToSend, remoteActor.ref) - + // remote actor receives message remoteActor.expectMsg(messageToSend) - + // remote actor sends a backpressure message remoteActor.reply(BackPressure(messageToSend)) - + // remote actor expects request again after backpressureTimeout remoteActor.expectMsg(1.second + margin, messageToSend) - + // remote actor replies remoteActor.reply("world") - + // delegate actor receives response delegateActor.expectMsg("world") - + // remote actor doesn't receives new messages remoteActor.expectNoMessage() - + // Wait long enough that to make sure that we won't receive a ServiceUnreachable message, meaning the timeout timer // has been cancelled. Note that it is the responsibility of the actor to cancel it, the RobustClientHelper does not // handle that part. @@ -57,7 +57,7 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc val remoteActor = TestProbe() val delegateActor = TestProbe() - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 20 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) @@ -68,13 +68,13 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc // remote actor receives message remoteActor.expectMsg(messageToSend) - + // remote actor replies remoteActor.reply("world") - + // delegate receives response delegateActor.expectMsg("world") - + // remote actor doesn't receives new messages remoteActor.expectNoMessage() delegateActor.expectNoMessage() @@ -84,7 +84,7 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc val remoteActor = TestProbe() val delegateActor = TestProbe() - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 2 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) @@ -109,9 +109,9 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc it should "reset timeout when backpressured is received" in { val remoteActor = TestProbe() val delegateActor = TestProbe() - + val margin = 1 second - val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2D, 0D) + val backoff = SimpleExponentialBackoff(1.second, 10.seconds, 2d, 0d) val noResponseTimeout = 3 seconds val testActor = TestActorRef(new TestActor(delegateActor.ref, backoff, noResponseTimeout)) @@ -141,32 +141,31 @@ class RobustClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matc delegateActor.expectNoMessage(4 seconds) } - private [actor] object TestActor { + private[actor] object TestActor { case class TestMessage(v: String) case object ServiceUnreachable } - private class TestActor(delegateTo: ActorRef, - backoff: Backoff, - noResponseTimeout: FiniteDuration) extends Actor with ActorLogging with RobustClientHelper { + private class TestActor(delegateTo: ActorRef, backoff: Backoff, noResponseTimeout: FiniteDuration) + extends Actor + with ActorLogging + with RobustClientHelper { override def initialBackoff(): Backoff = backoff context.become(robustReceive orElse receive) var messageSent: Any = _ - - override def receive: Receive = { - case message => - cancelTimeout(messageSent) - delegateTo ! message + + override def receive: Receive = { case message => + cancelTimeout(messageSent) + delegateTo ! message } - + def sendMessage(message: Any, to: ActorRef) = { messageSent = message robustSend(message, to, noResponseTimeout) } - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = delegateTo ! TestActor.ServiceUnreachable - } } } diff --git a/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala b/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala index a7d42d7ffdd..b45a5e6fba7 100644 --- a/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala +++ b/core/src/test/scala/cromwell/core/actor/StreamActorHelperSpec.scala @@ -13,10 +13,9 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.ExecutionContext - class StreamActorHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers with ImplicitSender { behavior of "StreamActorHelper" - + implicit val materializer = ActorMaterializer() it should "catch EnqueueResponse message" in { @@ -26,7 +25,7 @@ class StreamActorHelperSpec extends TestKitSuite with AnyFlatSpecLike with Match expectMsg("hello") system stop actor } - + it should "send a backpressure message when messages are dropped by the queue" in { val actor = TestActorRef(new TestStreamActor(1)) val command = new TestStreamActorCommand @@ -50,14 +49,19 @@ class StreamActorHelperSpec extends TestKitSuite with AnyFlatSpecLike with Match } } - private object TestStreamActor { class TestStreamActorCommand - case class TestStreamActorContext(request: TestStreamActorCommand, replyTo: ActorRef, override val clientContext: Option[Any]) extends StreamContext + case class TestStreamActorContext(request: TestStreamActorCommand, + replyTo: ActorRef, + override val clientContext: Option[Any] + ) extends StreamContext } -private class TestStreamActor(queueSize: Int)(implicit override val materializer: ActorMaterializer) extends Actor with ActorLogging with StreamActorHelper[TestStreamActorContext] { - +private class TestStreamActor(queueSize: Int)(implicit override val materializer: ActorMaterializer) + extends Actor + with ActorLogging + with StreamActorHelper[TestStreamActorContext] { + override protected def actorReceive: Receive = { case command: TestStreamActorCommand => val replyTo = sender() @@ -69,8 +73,9 @@ private class TestStreamActor(queueSize: Int)(implicit override val materializer sendToStream(commandContext) } - override protected val streamSource = Source.queue[TestStreamActorContext](queueSize, OverflowStrategy.dropNew) - .map{ ("hello", _) } + override protected val streamSource = Source + .queue[TestStreamActorContext](queueSize, OverflowStrategy.dropNew) + .map(("hello", _)) - override implicit def ec: ExecutionContext = context.dispatcher + implicit override def ec: ExecutionContext = context.dispatcher } diff --git a/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala b/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala index 82bc6bacb25..dbd8bd4b82c 100644 --- a/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala +++ b/core/src/test/scala/cromwell/core/callcaching/HashKeySpec.scala @@ -4,7 +4,6 @@ import common.assertion.CromwellTimeoutSpec import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class HashKeySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "HashKey" should "produce consistent key value" in { @@ -20,7 +19,7 @@ class HashKeySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { HashKey("output", "String myOutput"), HashKey("runtime attribute", "docker") ) - + keys map { _.key } should contain theSameElementsAs Set( "command template", "backend name", @@ -34,5 +33,5 @@ class HashKeySpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "runtime attribute: docker" ) } - + } diff --git a/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala b/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala index 8e958cce518..caacfd54ed8 100644 --- a/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala +++ b/core/src/test/scala/cromwell/core/filesystem/CromwellFileSystemsSpec.scala @@ -15,13 +15,12 @@ import scala.concurrent.ExecutionContext class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { behavior of "CromwellFileSystems" - val globalConfig = ConfigFactory.parseString( - """ - |filesystems { - | fs1.class = "cromwell.core.path.MockPathBuilderFactory" - | fs2.class = "cromwell.core.path.MockPathBuilderFactory" - | fs3.class = "cromwell.core.filesystem.MockNotPathBuilderFactory" - |} + val globalConfig = ConfigFactory.parseString(""" + |filesystems { + | fs1.class = "cromwell.core.path.MockPathBuilderFactory" + | fs2.class = "cromwell.core.path.MockPathBuilderFactory" + | fs3.class = "cromwell.core.filesystem.MockNotPathBuilderFactory" + |} """.stripMargin) val cromwellFileSystems = new CromwellFileSystems(globalConfig) @@ -29,12 +28,11 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with it should "build factory builders and factories for valid configuration" in { cromwellFileSystems.factoryBuilders.keySet shouldBe Set("fs1", "fs2", "fs3") - val factoriesConfig = ConfigFactory.parseString( - """ - |filesystems { - | fs1.somekey = "somevalue" - | fs2.someotherkey = "someothervalue" - |} + val factoriesConfig = ConfigFactory.parseString(""" + |filesystems { + | fs1.somekey = "somevalue" + | fs2.someotherkey = "someothervalue" + |} """.stripMargin) val pathFactories = cromwellFileSystems.factoriesFromConfig(factoriesConfig) @@ -48,16 +46,16 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with } it should "build singleton instance if specified" in { - val rootConf = ConfigFactory.parseString( - """ - |filesystems { - | fs1 { - | class = "cromwell.core.filesystem.MockPathBuilderFactoryCustomSingletonConfig" - | global { - | class = "cromwell.core.filesystem.MockSingletonConfig" - | } - | } - |} + val rootConf = + ConfigFactory.parseString(""" + |filesystems { + | fs1 { + | class = "cromwell.core.filesystem.MockPathBuilderFactoryCustomSingletonConfig" + | global { + | class = "cromwell.core.filesystem.MockSingletonConfig" + | } + | } + |} """.stripMargin) val cromwellFileSystems = new CromwellFileSystems(rootConf) @@ -69,21 +67,33 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with val factory2 = cromwellFileSystems.buildFactory("fs1", ConfigFactory.empty) // The singleton configs should be the same for different factories - assert(factory1.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig == - factory2.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig) + assert( + factory1.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig == + factory2.toOption.get.asInstanceOf[MockPathBuilderFactoryCustomSingletonConfig].singletonConfig + ) } List( - ("if the filesystem does not exist", "filesystems.fs4.key = value", NonEmptyList.one("Cannot find a filesystem with name fs4 in the configuration. Available filesystems: fs1, fs2, fs3")), - ("if the config is invalid", "filesystems.fs1 = true", NonEmptyList.one("Invalid filesystem backend configuration for fs1")), - ("the class is not a PathBuilderFactory", "filesystems.fs3.key = value", NonEmptyList.one("The filesystem class for fs3 is not an instance of PathBuilderFactory")) - ) foreach { - case (description, config, expected) => - it should s"fail to build factories $description" in { - val result = cromwellFileSystems.factoriesFromConfig(ConfigFactory.parseString(config)) - result.isLeft shouldBe true - result.swap.toOption.get shouldBe expected - } + ("if the filesystem does not exist", + "filesystems.fs4.key = value", + NonEmptyList.one( + "Cannot find a filesystem with name fs4 in the configuration. Available filesystems: fs1, fs2, fs3" + ) + ), + ("if the config is invalid", + "filesystems.fs1 = true", + NonEmptyList.one("Invalid filesystem backend configuration for fs1") + ), + ("the class is not a PathBuilderFactory", + "filesystems.fs3.key = value", + NonEmptyList.one("The filesystem class for fs3 is not an instance of PathBuilderFactory") + ) + ) foreach { case (description, config, expected) => + it should s"fail to build factories $description" in { + val result = cromwellFileSystems.factoriesFromConfig(ConfigFactory.parseString(config)) + result.isLeft shouldBe true + result.swap.toOption.get shouldBe expected + } } val classNotFoundException = AggregatedMessageException( @@ -93,7 +103,9 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with val wrongSignatureException = AggregatedMessageException( "Failed to initialize Cromwell filesystems", - List("Class cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature for filesystem fs1 does not have the required constructor signature: (com.typesafe.config.Config, com.typesafe.config.Config)") + List( + "Class cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature for filesystem fs1 does not have the required constructor signature: (com.typesafe.config.Config, com.typesafe.config.Config)" + ) ) val invalidConfigException = AggregatedMessageException( @@ -110,13 +122,15 @@ class CromwellFileSystemsSpec extends AnyFlatSpec with CromwellTimeoutSpec with ("is invalid", "filesystems.gcs = true", invalidConfigException), ("is missing class fields", "filesystems.fs1.notclass = hello", missingClassFieldException), ("can't find class", "filesystems.fs1.class = do.not.exists", classNotFoundException), - ("has invalid class signature", "filesystems.fs1.class = cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature", wrongSignatureException) - ) foreach { - case (description, config, expected) => - it should s"fail if global filesystems config $description" in { - val ex = the[Exception] thrownBy { new CromwellFileSystems(ConfigFactory.parseString(config)) } - ex shouldBe expected - } + ("has invalid class signature", + "filesystems.fs1.class = cromwell.core.filesystem.MockPathBuilderFactoryWrongSignature", + wrongSignatureException + ) + ) foreach { case (description, config, expected) => + it should s"fail if global filesystems config $description" in { + val ex = the[Exception] thrownBy new CromwellFileSystems(ConfigFactory.parseString(config)) + ex shouldBe expected + } } } @@ -124,6 +138,10 @@ class MockPathBuilderFactoryWrongSignature() class MockNotPathBuilderFactory(globalConfig: Config, val instanceConfig: Config) class MockSingletonConfig(config: Config) -class MockPathBuilderFactoryCustomSingletonConfig(globalConfig: Config, val instanceConfig: Config, val singletonConfig: MockSingletonConfig) extends cromwell.core.path.PathBuilderFactory { - override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = throw new UnsupportedOperationException +class MockPathBuilderFactoryCustomSingletonConfig(globalConfig: Config, + val instanceConfig: Config, + val singletonConfig: MockSingletonConfig +) extends cromwell.core.path.PathBuilderFactory { + override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = + throw new UnsupportedOperationException } diff --git a/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala b/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala index 83521ea3432..07ad722ead2 100644 --- a/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala +++ b/core/src/test/scala/cromwell/core/io/AsyncIoSpec.scala @@ -95,7 +95,7 @@ class AsyncIoSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers { } // Honor swallow exception false - //noinspection RedundantDefaultArgument + // noinspection RedundantDefaultArgument recoverToSucceededIf[NoSuchFileException] { testActor.underlyingActor.asyncIo.deleteAsync(testPath, swallowIoExceptions = false) } @@ -103,8 +103,8 @@ class AsyncIoSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers { it should "handle command creation errors asynchronously" in { val partialIoCommandBuilder = new PartialIoCommandBuilder { - override def existsCommand: PartialFunction[Path, Try[IoExistsCommand]] = { - case _ => Failure(new Exception("everything's fine, I am an expected exists fail") with NoStackTrace) + override def existsCommand: PartialFunction[Path, Try[IoExistsCommand]] = { case _ => + Failure(new Exception("everything's fine, I am an expected exists fail") with NoStackTrace) } } val testActor = @@ -120,10 +120,11 @@ class AsyncIoSpec extends TestKitSuite with AsyncFlatSpecLike with Matchers { private class AsyncIoTestActor(override val ioActor: ActorRef, override val ioCommandBuilder: IoCommandBuilder = DefaultIoCommandBuilder - ) extends Actor with ActorLogging with AsyncIoActorClient { + ) extends Actor + with ActorLogging + with AsyncIoActorClient { - override def receive: Receive = { - case _ => + override def receive: Receive = { case _ => } } diff --git a/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala b/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala index f77df6fa011..ed07aeb3fc5 100644 --- a/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala +++ b/core/src/test/scala/cromwell/core/io/IoClientHelperSpec.scala @@ -21,13 +21,14 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers it should "intercept IoAcks and cancel timers" in { val ioActorProbe = TestProbe() val delegateProbe = TestProbe() - val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2D, 0D) + val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2d, 0d) val noResponseTimeout = 3 seconds - val testActor = TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) + val testActor = + TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) val command = DefaultIoSizeCommand(mock[Path]) - val response = IoSuccess(command, 5) + val response = IoSuccess(command, 5L) // Send the command testActor.underlyingActor.sendMessage(command) @@ -51,14 +52,15 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers it should "intercept IoAcks and cancel timers for a command with context" in { val ioActorProbe = TestProbe() val delegateProbe = TestProbe() - val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2D, 0D) + val backoff = SimpleExponentialBackoff(100 seconds, 10.hours, 2d, 0d) val noResponseTimeout = 3 seconds - val testActor = TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) + val testActor = + TestActorRef(new IoClientHelperTestActor(ioActorProbe.ref, delegateProbe.ref, backoff, noResponseTimeout)) val commandContext = "context" val command = DefaultIoSizeCommand(mock[Path]) - val response = IoSuccess(command, 5) + val response = IoSuccess(command, 5L) // Send the command testActor.underlyingActor.sendMessageWithContext(commandContext, command) @@ -71,7 +73,7 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers // delegate should receive the response delegateProbe.expectMsgPF(1 second) { - case (contextReceived, responseReceived) if contextReceived == "context" && responseReceived == response => + case (contextReceived, responseReceived) if contextReceived == "context" && responseReceived == response => } // And nothing else, meaning the timeout timer has been cancelled @@ -84,9 +86,12 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers private case object ServiceUnreachable private class IoClientHelperTestActor(override val ioActor: ActorRef, - delegateTo: ActorRef, - backoff: Backoff, - noResponseTimeout: FiniteDuration) extends Actor with ActorLogging with IoClientHelper { + delegateTo: ActorRef, + backoff: Backoff, + noResponseTimeout: FiniteDuration + ) extends Actor + with ActorLogging + with IoClientHelper { implicit val ioCommandBuilder: DefaultIoCommandBuilder.type = DefaultIoCommandBuilder @@ -94,21 +99,18 @@ class IoClientHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers context.become(ioReceive orElse receive) - override def receive: Receive = { - case message => delegateTo ! message + override def receive: Receive = { case message => + delegateTo ! message } - def sendMessage(command: IoCommand[_]): Unit = { + def sendMessage(command: IoCommand[_]): Unit = sendIoCommandWithCustomTimeout(command, noResponseTimeout) - } - def sendMessageWithContext(context: Any, command: IoCommand[_]): Unit = { + def sendMessageWithContext(context: Any, command: IoCommand[_]): Unit = sendIoCommandWithContext(command, context, noResponseTimeout) - } - override protected def onTimeout(message: Any, to: ActorRef): Unit = { + override protected def onTimeout(message: Any, to: ActorRef): Unit = delegateTo ! ServiceUnreachable - } } } diff --git a/core/src/test/scala/cromwell/core/labels/LabelSpec.scala b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala index 6bf4e046ed2..bdb443a91ac 100644 --- a/core/src/test/scala/cromwell/core/labels/LabelSpec.scala +++ b/core/src/test/scala/cromwell/core/labels/LabelSpec.scala @@ -29,7 +29,7 @@ class LabelSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { "11f2468c-39d6-4be3-85c8-32735c01e66b", "", "!@#$%^&*()_+={}[]:;'<>?,./`~", - "now valid 255 character value-at vero eosd accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa", + "now valid 255 character value-at vero eosd accusamus et iusto odio dignissimos ducimus qui blanditiis praesentium voluptatum deleniti atque corrupti quos dolores et quas molestias excepturi sint occaecati cupiditate non provident, similique sunt in culpa" ) val badLabelKeys = List( diff --git a/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala b/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala index d40456e0128..d3d99289a35 100644 --- a/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala +++ b/core/src/test/scala/cromwell/core/logging/LoggerWrapperSpec.scala @@ -13,8 +13,12 @@ import org.slf4j.Logger import org.slf4j.event.Level import common.mock.MockSugar -class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers with MockSugar - with TableDrivenPropertyChecks { +class LoggerWrapperSpec + extends AnyFlatSpec + with CromwellTimeoutSpec + with Matchers + with MockSugar + with TableDrivenPropertyChecks { behavior of "LoggerWrapper" @@ -25,7 +29,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche "slf4jMessages", "akkaMessages" ), - ( "log error with no args", _.error("Hello {} {} {} {}"), @@ -80,7 +83,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.ERROR, List("tag: Hello {} {} {} {}", "arg1", exception))), List(AkkaMessage(Logging.ErrorLevel, s"tag: Hello arg1 {} {} {}", Option(exception))) ), - ( "log warn with no args", _.warn("Hello {} {} {} {}"), @@ -123,7 +125,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.WARN, List("tag: Hello {} {} {} {}", exception))), List(AkkaMessage(Logging.WarningLevel, s"tag: Hello {} {} {} {}\n$exceptionMessage")) ), - ( "log info with no args", _.info("Hello {} {} {} {}"), @@ -138,7 +139,7 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche ), ( "log info with one arg", - _.info("Hello {} {} {} {}", arg ="arg1"), + _.info("Hello {} {} {} {}", arg = "arg1"), List(Slf4jMessage(Level.INFO, List("tag: Hello {} {} {} {}", "arg1"))), List(AkkaMessage(Logging.InfoLevel, "tag: Hello arg1 {} {} {}")) ), @@ -166,7 +167,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.INFO, List("tag: Hello {} {} {} {}", exception))), List(AkkaMessage(Logging.InfoLevel, s"tag: Hello {} {} {} {}\n$exceptionMessage")) ), - ( "log debug with no args", _.debug("Hello {} {} {} {}"), @@ -181,7 +181,7 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche ), ( "log debug with one arg", - _.debug("Hello {} {} {} {}", argument ="arg1"), + _.debug("Hello {} {} {} {}", argument = "arg1"), List(Slf4jMessage(Level.DEBUG, List("tag: Hello {} {} {} {}", "arg1"))), List(AkkaMessage(Logging.DebugLevel, "tag: Hello arg1 {} {} {}")) ), @@ -209,7 +209,6 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche List(Slf4jMessage(Level.DEBUG, List("tag: Hello {} {} {} {}", exception))), List(AkkaMessage(Logging.DebugLevel, s"tag: Hello {} {} {} {}\n$exceptionMessage")) ), - ( "log trace with no args", _.trace("Hello {} {} {} {}"), @@ -260,40 +259,36 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche var actualAkkaMessages = List.empty[AkkaMessage] - def toList(arguments: Any): List[Any] = { + def toList(arguments: Any): List[Any] = arguments match { case array: Array[_] => toLastFlattened(array) case seq: Seq[_] => seq.toList case any => List(any) } - } /* - * Flatten the last element of the array if the last element is itself an array. - * - * org.mockito.ArgumentMatchers#anyVararg() is deprecated, but works, sending in an empty array in the tail - * position. If we tried to use org.mockito.ArgumentMatchers#any(), it ends up mocking the wrong overloaded - * method. At each logging level there are two methods with very similar signatures: - * - * cromwell.core.logging.LoggerWrapper.error(pattern: String, arguments: AnyRef*) - * cromwell.core.logging.LoggerWrapper.error(pattern: String, arg: Any) - * - * As is, the Any vs. AnyRef overloads are barely dodging the issue https://issues.scala-lang.org/browse/SI-2991. - */ - def toLastFlattened(array: Array[_]): List[Any] = { + * Flatten the last element of the array if the last element is itself an array. + * + * org.mockito.ArgumentMatchers#anyVararg() is deprecated, but works, sending in an empty array in the tail + * position. If we tried to use org.mockito.ArgumentMatchers#any(), it ends up mocking the wrong overloaded + * method. At each logging level there are two methods with very similar signatures: + * + * cromwell.core.logging.LoggerWrapper.error(pattern: String, arguments: AnyRef*) + * cromwell.core.logging.LoggerWrapper.error(pattern: String, arg: Any) + * + * As is, the Any vs. AnyRef overloads are barely dodging the issue https://issues.scala-lang.org/browse/SI-2991. + */ + def toLastFlattened(array: Array[_]): List[Any] = array.toList.reverse match { case (array: Array[_]) :: tail => tail.reverse ++ array.toList case other => other.reverse } - } - def updateSlf4jMessages(level: Level, arguments: Any): Unit = { + def updateSlf4jMessages(level: Level, arguments: Any): Unit = actualSlf4jMessages :+= Slf4jMessage(level, toList(arguments)) - } - def updateAkkaMessages(logLevel: LogLevel, message: String, causeOption: Option[Throwable] = None): Unit = { + def updateAkkaMessages(logLevel: LogLevel, message: String, causeOption: Option[Throwable] = None): Unit = actualAkkaMessages :+= AkkaMessage(logLevel, message, causeOption) - } val mockLogger = mock[Logger] @@ -333,25 +328,20 @@ class LoggerWrapperSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matche override val isInfoEnabled: Boolean = true override val isDebugEnabled: Boolean = true - override protected def notifyError(message: String): Unit = { + override protected def notifyError(message: String): Unit = updateAkkaMessages(Logging.ErrorLevel, message) - } - override protected def notifyError(cause: Throwable, message: String): Unit = { + override protected def notifyError(cause: Throwable, message: String): Unit = updateAkkaMessages(Logging.ErrorLevel, message, Option(cause)) - } - override protected def notifyWarning(message: String): Unit = { + override protected def notifyWarning(message: String): Unit = updateAkkaMessages(Logging.WarningLevel, message) - } - override protected def notifyInfo(message: String): Unit = { + override protected def notifyInfo(message: String): Unit = updateAkkaMessages(Logging.InfoLevel, message) - } - override protected def notifyDebug(message: String): Unit = { + override protected def notifyDebug(message: String): Unit = updateAkkaMessages(Logging.DebugLevel, message) - } } val wrapper = new LoggerWrapper { diff --git a/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala b/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala index 63a5a8604cf..5235f711063 100644 --- a/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala +++ b/core/src/test/scala/cromwell/core/path/DefaultPathBuilderSpec.scala @@ -6,7 +6,12 @@ import org.scalatest.flatspec.AnyFlatSpecLike import org.scalatest.matchers.should.Matchers import org.scalatest.prop.Tables.Table -class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers with PathBuilderSpecUtils with TestFileUtil { +class DefaultPathBuilderSpec + extends Suite + with AnyFlatSpecLike + with Matchers + with PathBuilderSpecUtils + with TestFileUtil { private val pwd = BetterFileMethods.Cmds.pwd private val parentOption = Option(pwd.parent) @@ -57,7 +62,6 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi ) private def goodPaths = Seq( - // Normal paths, not normalized GoodPath( @@ -72,8 +76,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "world", getFileName = "world", getNameCount = 2, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a relative path", path = "hello/world", @@ -86,8 +90,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "world", getFileName = "world", getNameCount = 2, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path with spaces", path = "/hello/world/with spaces", @@ -100,8 +104,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "with spaces", getFileName = "with spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path with encode spaces", path = "/hello/world/encoded%20spaces", @@ -114,8 +118,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "encoded%20spaces", getFileName = "encoded%20spaces", getNameCount = 3, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path with non-ascii characters", path = "/hello/world/with non ascii £€", @@ -128,7 +132,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "with non ascii £€", getFileName = "with non ascii £€", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // Special paths @@ -144,8 +149,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = "", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path from /", path = "/", @@ -158,8 +163,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path from .", path = ".", @@ -172,8 +177,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = ".", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path from ..", path = "..", @@ -186,8 +191,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = parentName, getFileName = "..", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a path including .", path = "/hello/world/with/./dots", @@ -200,8 +205,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 5, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path including ..", path = "/hello/world/with/../dots", @@ -214,7 +219,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 5, - isAbsolute = true), + isAbsolute = true + ), // Normalized @@ -230,8 +236,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = "", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a normalized path from /", path = "/", @@ -244,8 +250,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "", getFileName = null, getNameCount = 0, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a normalized path from .", path = ".", @@ -258,8 +264,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = pwdName, getFileName = "", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a normalized path from ..", path = "..", @@ -272,8 +278,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = parentName, getFileName = "..", getNameCount = 1, - isAbsolute = false), - + isAbsolute = false + ), GoodPath( description = "a normalized path including a .", path = "/hello/world/with/./dots", @@ -286,8 +292,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 4, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a normalized path including ..", path = "/hello/world/with/../dots", @@ -300,7 +306,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "dots", getFileName = "dots", getNameCount = 3, - isAbsolute = true), + isAbsolute = true + ), // URI @@ -316,8 +323,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "world", getFileName = "world", getNameCount = 2, - isAbsolute = true), - + isAbsolute = true + ), GoodPath( description = "a path from a file uri with encoded spaces", path = "file:///hello/world/encoded%20spaces", @@ -330,7 +337,8 @@ class DefaultPathBuilderSpec extends Suite with AnyFlatSpecLike with Matchers wi name = "encoded%20spaces", getFileName = "encoded%20spaces", getNameCount = 3, - isAbsolute = true) + isAbsolute = true + ) ) private def badPaths = Seq( diff --git a/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala b/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala index 7fea4f7b9f3..7119a62b1f6 100644 --- a/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala +++ b/core/src/test/scala/cromwell/core/path/PathBuilderFactorySpec.scala @@ -12,24 +12,28 @@ import scala.concurrent.{ExecutionContext, Future} class PathBuilderFactorySpec extends TestKitSuite with AnyFlatSpecLike with ScalaFutures with Matchers { behavior of "PathBuilderFactory" implicit val ec = system.dispatcher - + it should "sort factories when instantiating path builders" in { val factory1 = new MockPathBuilderFactory(ConfigFactory.empty(), ConfigFactory.parseString("name=factory1")) val factory2 = new MockPathBuilderFactory(ConfigFactory.empty(), ConfigFactory.parseString("name=factory2")) PathBuilderFactory - .instantiatePathBuilders(List(DefaultPathBuilderFactory, factory1, factory2), WorkflowOptions.empty).map({ pathBuilders => - pathBuilders.last shouldBe DefaultPathBuilder - // check that the order of the other factories has not been changed - pathBuilders.map(_.name) shouldBe List("factory1", "factory2", DefaultPathBuilder.name) - }).futureValue + .instantiatePathBuilders(List(DefaultPathBuilderFactory, factory1, factory2), WorkflowOptions.empty) + .map { pathBuilders => + pathBuilders.last shouldBe DefaultPathBuilder + // check that the order of the other factories has not been changed + pathBuilders.map(_.name) shouldBe List("factory1", "factory2", DefaultPathBuilder.name) + } + .futureValue } } -class MockPathBuilderFactory(globalConfig: Config, val instanceConfig: Config) extends cromwell.core.path.PathBuilderFactory { - override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = Future.successful( - new PathBuilder { - override def name = instanceConfig.getString("name") - override def build(pathAsString: String) = throw new UnsupportedOperationException - } - ) +class MockPathBuilderFactory(globalConfig: Config, val instanceConfig: Config) + extends cromwell.core.path.PathBuilderFactory { + override def withOptions(options: WorkflowOptions)(implicit as: ActorSystem, ec: ExecutionContext) = + Future.successful( + new PathBuilder { + override def name = instanceConfig.getString("name") + override def build(pathAsString: String) = throw new UnsupportedOperationException + } + ) } diff --git a/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala b/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala index 92bfd77233f..324928d2cb4 100644 --- a/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala +++ b/core/src/test/scala/cromwell/core/path/PathBuilderSpecUtils.scala @@ -17,7 +17,8 @@ case class GoodPath(description: String, name: String, getFileName: String, getNameCount: Int, - isAbsolute: Boolean) + isAbsolute: Boolean +) case class BadPath(description: String, path: String, exceptionMessage: String) @@ -29,7 +30,8 @@ trait PathBuilderSpecUtils { def truncateCommonRoots(builder: => PathBuilder, pathsToTruncate: TableFor3[String, String, String], - tag: Tag = PathBuilderSpecUtils.PathTest): Unit = { + tag: Tag = PathBuilderSpecUtils.PathTest + ): Unit = { behavior of s"PathCopier" it should "truncate common roots" taggedAs tag in { diff --git a/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala b/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala index 58fa597dc03..e5d947da83d 100644 --- a/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala +++ b/core/src/test/scala/cromwell/core/retry/BackoffSpec.scala @@ -18,11 +18,10 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { .setInitialIntervalMillis(1.second.toMillis.toInt) .setMaxIntervalMillis(2.seconds.toMillis.toInt) .setMaxElapsedTimeMillis(Integer.MAX_VALUE) - .setRandomizationFactor(0D) + .setRandomizationFactor(0d) .build() ) - exponentialBackoff.backoffMillis shouldBe 3.seconds.toMillis exponentialBackoff.next.backoffMillis shouldBe 1.second.toMillis } @@ -33,7 +32,7 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { .setInitialIntervalMillis(1.second.toMillis.toInt) .setMaxIntervalMillis(2.seconds.toMillis.toInt) .setMaxElapsedTimeMillis(Integer.MAX_VALUE) - .setRandomizationFactor(0D) + .setRandomizationFactor(0d) .build() ).backoffMillis shouldBe 1.second.toMillis } @@ -46,7 +45,7 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { .setInitialIntervalMillis(1.second.toMillis.toInt) .setMaxIntervalMillis(2.seconds.toMillis.toInt) .setMaxElapsedTimeMillis(Integer.MAX_VALUE) - .setRandomizationFactor(0D) + .setRandomizationFactor(0d) .build() ) } @@ -54,19 +53,19 @@ class BackoffSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { it should "parse config" in { val config = ConfigFactory.parseMap( - Map( + Map[String, Any]( "min" -> "5 seconds", "max" -> "30 seconds", - "multiplier" -> 6D, - "randomization-factor" -> 0D + "multiplier" -> 6d, + "randomization-factor" -> 0d ).asJava ) val backoff = SimpleExponentialBackoff(config) backoff.googleBackoff.getCurrentIntervalMillis shouldBe 5.seconds.toMillis.toInt backoff.googleBackoff.getMaxIntervalMillis shouldBe 30.seconds.toMillis.toInt - backoff.googleBackoff.getMultiplier shouldBe 6D - backoff.googleBackoff.getRandomizationFactor shouldBe 0D + backoff.googleBackoff.getMultiplier shouldBe 6d + backoff.googleBackoff.getRandomizationFactor shouldBe 0d } } diff --git a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala index 83516f3328b..134b1c1db25 100644 --- a/core/src/test/scala/cromwell/core/retry/RetrySpec.scala +++ b/core/src/test/scala/cromwell/core/retry/RetrySpec.scala @@ -16,7 +16,7 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca var counter: Int = n - def doIt(): Future[Int] = { + def doIt(): Future[Int] = if (counter == 0) Future.successful(9) else { @@ -24,23 +24,22 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca val ex = if (counter <= transients) new TransientException else new IllegalArgumentException("Failed") Future.failed(ex) } - } } - implicit val defaultPatience: PatienceConfig = PatienceConfig(timeout = Span(30, Seconds), interval = Span(100, Millis)) + implicit val defaultPatience: PatienceConfig = + PatienceConfig(timeout = Span(30, Seconds), interval = Span(100, Millis)) private def runRetry(retries: Int, work: MockWork, isTransient: Throwable => Boolean = Retry.throwableToFalse, - isFatal: Throwable => Boolean = Retry.throwableToFalse): Future[Int] = { - + isFatal: Throwable => Boolean = Retry.throwableToFalse + ): Future[Int] = withRetry( f = () => work.doIt(), maxRetries = Option(retries), isTransient = isTransient, isFatal = isFatal ) - } "Retry" should "retry a function until it works" in { val work = new MockWork(2) @@ -53,7 +52,7 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca it should "fail if it hits the max retry count" in { whenReady(runRetry(1, new MockWork(3)).failed) { x => - x shouldBe an [CromwellFatalException] + x shouldBe an[CromwellFatalException] } } @@ -61,18 +60,19 @@ class RetrySpec extends TestKitSuite with AnyFlatSpecLike with Matchers with Sca val work = new MockWork(3) whenReady(runRetry(3, work, isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException]).failed) { x => - x shouldBe an [CromwellFatalException] + x shouldBe an[CromwellFatalException] work.counter shouldBe 2 } val work2 = new MockWork(4, 2) val retry = runRetry(4, - work2, - isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException], - isTransient = (t: Throwable) => t.isInstanceOf[TransientException]) + work2, + isFatal = (t: Throwable) => t.isInstanceOf[IllegalArgumentException], + isTransient = (t: Throwable) => t.isInstanceOf[TransientException] + ) whenReady(retry.failed) { x => - x shouldBe an [CromwellFatalException] + x shouldBe an[CromwellFatalException] work2.counter shouldBe 3 } } diff --git a/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala b/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala index 8db76caab1d..fb347e43441 100644 --- a/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala +++ b/core/src/test/scala/cromwell/core/simpleton/WomValueBuilderSpec.scala @@ -23,110 +23,165 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc case class SimpletonConversion(name: String, womValue: WomValue, simpletons: Seq[WomValueSimpleton]) val simpletonConversions = List( SimpletonConversion("foo", WomString("none"), List(WomValueSimpleton("foo", WomString("none")))), - SimpletonConversion("bar", WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))), List(WomValueSimpleton("bar[0]", WomInteger(1)), WomValueSimpleton("bar[1]", WomInteger(2)))), + SimpletonConversion( + "bar", + WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))), + List(WomValueSimpleton("bar[0]", WomInteger(1)), WomValueSimpleton("bar[1]", WomInteger(2))) + ), SimpletonConversion("empty_array", WomArray(WomArrayType(WomIntegerType), List.empty), List()), SimpletonConversion( "baz", - WomArray(WomArrayType(WomArrayType(WomIntegerType)), List( - WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), - WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))))), - List(WomValueSimpleton("baz[0][0]", WomInteger(0)), WomValueSimpleton("baz[0][1]", WomInteger(1)), WomValueSimpleton("baz[1][0]", WomInteger(2)), WomValueSimpleton("baz[1][1]", WomInteger(3))) + WomArray( + WomArrayType(WomArrayType(WomIntegerType)), + List(WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), + WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))) + ) + ), + List( + WomValueSimpleton("baz[0][0]", WomInteger(0)), + WomValueSimpleton("baz[0][1]", WomInteger(1)), + WomValueSimpleton("baz[1][0]", WomInteger(2)), + WomValueSimpleton("baz[1][1]", WomInteger(3)) + ) ), SimpletonConversion( "map", - WomMap(WomMapType(WomStringType, WomStringType), Map( - WomString("foo") -> WomString("foo"), - WomString("bar") -> WomString("bar"))), + WomMap(WomMapType(WomStringType, WomStringType), + Map(WomString("foo") -> WomString("foo"), WomString("bar") -> WomString("bar")) + ), List(WomValueSimpleton("map:foo", WomString("foo")), WomValueSimpleton("map:bar", WomString("bar"))) ), SimpletonConversion( "mapOfMaps", - WomMap(WomMapType(WomStringType, WomMapType(WomStringType, WomStringType)), Map( - WomString("foo") -> WomMap(WomMapType(WomStringType, WomStringType), Map(WomString("foo2") -> WomString("foo"))), - WomString("bar") ->WomMap(WomMapType(WomStringType, WomStringType), Map(WomString("bar2") -> WomString("bar"))))), - List(WomValueSimpleton("mapOfMaps:foo:foo2", WomString("foo")), WomValueSimpleton("mapOfMaps:bar:bar2", WomString("bar"))) + WomMap( + WomMapType(WomStringType, WomMapType(WomStringType, WomStringType)), + Map( + WomString("foo") -> WomMap(WomMapType(WomStringType, WomStringType), + Map(WomString("foo2") -> WomString("foo")) + ), + WomString("bar") -> WomMap(WomMapType(WomStringType, WomStringType), + Map(WomString("bar2") -> WomString("bar")) + ) + ) + ), + List(WomValueSimpleton("mapOfMaps:foo:foo2", WomString("foo")), + WomValueSimpleton("mapOfMaps:bar:bar2", WomString("bar")) + ) ), SimpletonConversion( "simplePair1", WomPair(WomInteger(1), WomString("hello")), - List(WomValueSimpleton("simplePair1:left", WomInteger(1)), WomValueSimpleton("simplePair1:right", WomString("hello"))) + List(WomValueSimpleton("simplePair1:left", WomInteger(1)), + WomValueSimpleton("simplePair1:right", WomString("hello")) + ) ), SimpletonConversion( "simplePair2", WomPair(WomString("left"), WomInteger(5)), - List(WomValueSimpleton("simplePair2:left", WomString("left")), WomValueSimpleton("simplePair2:right", WomInteger(5))) + List(WomValueSimpleton("simplePair2:left", WomString("left")), + WomValueSimpleton("simplePair2:right", WomInteger(5)) + ) ), SimpletonConversion( "pairOfPairs", - WomPair( - WomPair(WomInteger(1), WomString("one")), - WomPair(WomString("two"), WomInteger(2))), + WomPair(WomPair(WomInteger(1), WomString("one")), WomPair(WomString("two"), WomInteger(2))), List( WomValueSimpleton("pairOfPairs:left:left", WomInteger(1)), WomValueSimpleton("pairOfPairs:left:right", WomString("one")), WomValueSimpleton("pairOfPairs:right:left", WomString("two")), - WomValueSimpleton("pairOfPairs:right:right", WomInteger(2))) + WomValueSimpleton("pairOfPairs:right:right", WomInteger(2)) + ) ), SimpletonConversion( "pairOfArrayAndMap", WomPair( WomArray(WomArrayType(WomIntegerType), List(WomInteger(1), WomInteger(2))), - WomMap(WomMapType(WomStringType, WomIntegerType), Map(WomString("left") -> WomInteger(100), WomString("right") -> WomInteger(200)))), + WomMap(WomMapType(WomStringType, WomIntegerType), + Map(WomString("left") -> WomInteger(100), WomString("right") -> WomInteger(200)) + ) + ), List( WomValueSimpleton("pairOfArrayAndMap:left[0]", WomInteger(1)), WomValueSimpleton("pairOfArrayAndMap:left[1]", WomInteger(2)), WomValueSimpleton("pairOfArrayAndMap:right:left", WomInteger(100)), - WomValueSimpleton("pairOfArrayAndMap:right:right", WomInteger(200))) + WomValueSimpleton("pairOfArrayAndMap:right:right", WomInteger(200)) + ) ), SimpletonConversion( "mapOfArrays", - WomMap(WomMapType(WomStringType, WomArrayType(WomIntegerType)), Map( - WomString("foo") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), - WomString("bar") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))))), - List(WomValueSimpleton("mapOfArrays:foo[0]", WomInteger(0)), WomValueSimpleton("mapOfArrays:foo[1]", WomInteger(1)), - WomValueSimpleton("mapOfArrays:bar[0]", WomInteger(2)), WomValueSimpleton("mapOfArrays:bar[1]", WomInteger(3))) + WomMap( + WomMapType(WomStringType, WomArrayType(WomIntegerType)), + Map( + WomString("foo") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(0), WomInteger(1))), + WomString("bar") -> WomArray(WomArrayType(WomIntegerType), List(WomInteger(2), WomInteger(3))) + ) + ), + List( + WomValueSimpleton("mapOfArrays:foo[0]", WomInteger(0)), + WomValueSimpleton("mapOfArrays:foo[1]", WomInteger(1)), + WomValueSimpleton("mapOfArrays:bar[0]", WomInteger(2)), + WomValueSimpleton("mapOfArrays:bar[1]", WomInteger(3)) + ) ), SimpletonConversion( "escapology", - WomMap(WomMapType(WomStringType, WomStringType), Map( - WomString("foo[1]") -> WomString("foo"), - WomString("bar[[") -> WomString("bar"), - WomString("baz:qux") -> WomString("baz:qux"))), - List(WomValueSimpleton("escapology:foo\\[1\\]", WomString("foo")), + WomMap( + WomMapType(WomStringType, WomStringType), + Map(WomString("foo[1]") -> WomString("foo"), + WomString("bar[[") -> WomString("bar"), + WomString("baz:qux") -> WomString("baz:qux") + ) + ), + List( + WomValueSimpleton("escapology:foo\\[1\\]", WomString("foo")), WomValueSimpleton("escapology:bar\\[\\[", WomString("bar")), - WomValueSimpleton("escapology:baz\\:qux", WomString("baz:qux"))) + WomValueSimpleton("escapology:baz\\:qux", WomString("baz:qux")) + ) ), SimpletonConversion( "flat_object", - WomObject(Map( - "a" -> WomString("aardvark"), - "b" -> WomInteger(25), - "c" -> WomBoolean(false) - )), - List(WomValueSimpleton("flat_object:a", WomString("aardvark")), + WomObject( + Map( + "a" -> WomString("aardvark"), + "b" -> WomInteger(25), + "c" -> WomBoolean(false) + ) + ), + List( + WomValueSimpleton("flat_object:a", WomString("aardvark")), WomValueSimpleton("flat_object:b", WomInteger(25)), - WomValueSimpleton("flat_object:c", WomBoolean(false))) + WomValueSimpleton("flat_object:c", WomBoolean(false)) + ) ), SimpletonConversion( "object_with_array", - WomObject(Map( - "a" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("beetle"))) - )), + WomObject( + Map( + "a" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("beetle"))) + ) + ), List(WomValueSimpleton("object_with_array:a[0]", WomString("aardvark")), - WomValueSimpleton("object_with_array:a[1]", WomString("beetle"))) + WomValueSimpleton("object_with_array:a[1]", WomString("beetle")) + ) ), SimpletonConversion( "object_with_object", - WomObject(Map( - "a" -> WomObject(Map( - "aa" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), - "ab" -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) - )), - "b" -> WomObject(Map( - "ba" -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), - "bb" -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) - )) - )), + WomObject( + Map( + "a" -> WomObject( + Map( + "aa" -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), + "ab" -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) + ) + ), + "b" -> WomObject( + Map( + "ba" -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), + "bb" -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) + ) + ) + ) + ), List( WomValueSimpleton("object_with_object:a:aa[0]", WomString("aardvark")), WomValueSimpleton("object_with_object:a:aa[1]", WomString("aaron")), @@ -135,58 +190,59 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc WomValueSimpleton("object_with_object:b:ba[0]", WomString("baa")), WomValueSimpleton("object_with_object:b:ba[1]", WomString("battle")), WomValueSimpleton("object_with_object:b:bb[0]", WomString("bbrrrr")), - WomValueSimpleton("object_with_object:b:bb[1]", WomString("bb gun")), + WomValueSimpleton("object_with_object:b:bb[1]", WomString("bb gun")) ) ), /* - * Wom object representing a directory listing - * - a single file - * - a "maybe populated file" with some properties (checksum etc..) and secondary files: - * - another single file - * - a directory listing a single file - * - an unlisted directory - * - a glob file - * - an unlisted directory - * - a glob file - * - * Note: glob files technically are never simpletonized but as WomFiles they *can* be + * Wom object representing a directory listing + * - a single file + * - a "maybe populated file" with some properties (checksum etc..) and secondary files: + * - another single file + * - a directory listing a single file + * - an unlisted directory + * - a glob file + * - an unlisted directory + * - a glob file + * + * Note: glob files technically are never simpletonized but as WomFiles they *can* be */ SimpletonConversion( "directory", WomMaybeListedDirectory( Option("outerValueName"), - Option(List( - WomSingleFile("outerSingleFile"), - WomMaybeListedDirectory(Option("innerValueName"), Option(List(WomSingleFile("innerSingleFile")))), - WomMaybePopulatedFile( - Option("populatedInnerValueName"), - Option("innerChecksum"), - Option(10L), - Option("innerFormat"), - Option("innerContents"), - List( - WomSingleFile("populatedInnerSingleFile"), - WomMaybeListedDirectory(Option("innerDirectoryValueName"), Option(List(WomSingleFile("innerDirectorySingleFile")))), - WomUnlistedDirectory("innerUnlistedDirectory"), - WomGlobFile("innerGlobFile") - ) - ), - WomUnlistedDirectory("outerUnlistedDirectory"), - WomGlobFile("outerGlobFile") - ))), + Option( + List( + WomSingleFile("outerSingleFile"), + WomMaybeListedDirectory(Option("innerValueName"), Option(List(WomSingleFile("innerSingleFile")))), + WomMaybePopulatedFile( + Option("populatedInnerValueName"), + Option("innerChecksum"), + Option(10L), + Option("innerFormat"), + Option("innerContents"), + List( + WomSingleFile("populatedInnerSingleFile"), + WomMaybeListedDirectory(Option("innerDirectoryValueName"), + Option(List(WomSingleFile("innerDirectorySingleFile"))) + ), + WomUnlistedDirectory("innerUnlistedDirectory"), + WomGlobFile("innerGlobFile") + ) + ), + WomUnlistedDirectory("outerUnlistedDirectory"), + WomGlobFile("outerGlobFile") + ) + ) + ), List( WomValueSimpleton("directory:class", WomString("Directory")), WomValueSimpleton("directory:value", WomString("outerValueName")), - WomValueSimpleton("directory:listing[0]", WomSingleFile("outerSingleFile")), - WomValueSimpleton("directory:listing[1]:class", WomString("Directory")), WomValueSimpleton("directory:listing[1]:value", WomString("innerValueName")), WomValueSimpleton("directory:listing[1]:listing[0]", WomSingleFile("innerSingleFile")), - WomValueSimpleton("directory:listing[2]:class", WomString("File")), WomValueSimpleton("directory:listing[2]:value", WomString("populatedInnerValueName")), - WomValueSimpleton("directory:listing[2]:checksum", WomString("innerChecksum")), WomValueSimpleton("directory:listing[2]:size", WomInteger(10)), WomValueSimpleton("directory:listing[2]:format", WomString("innerFormat")), @@ -194,10 +250,11 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc WomValueSimpleton("directory:listing[2]:secondaryFiles[0]", WomSingleFile("populatedInnerSingleFile")), WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:class", WomString("Directory")), WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:value", WomString("innerDirectoryValueName")), - WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:listing[0]", WomSingleFile("innerDirectorySingleFile")), + WomValueSimpleton("directory:listing[2]:secondaryFiles[1]:listing[0]", + WomSingleFile("innerDirectorySingleFile") + ), WomValueSimpleton("directory:listing[2]:secondaryFiles[2]", WomUnlistedDirectory("innerUnlistedDirectory")), WomValueSimpleton("directory:listing[2]:secondaryFiles[3]", WomGlobFile("innerGlobFile")), - WomValueSimpleton("directory:listing[3]", WomUnlistedDirectory("outerUnlistedDirectory")), WomValueSimpleton("directory:listing[4]", WomGlobFile("outerGlobFile")) ) @@ -225,7 +282,9 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc it should "round trip everything together with no losses" in { - val wdlValues = (simpletonConversions map { case SimpletonConversion(name, womValue, _) => WomMocks.mockOutputPort(name, womValue.womType) -> womValue }).toMap + val wdlValues = (simpletonConversions map { case SimpletonConversion(name, womValue, _) => + WomMocks.mockOutputPort(name, womValue.womType) -> womValue + }).toMap val allSimpletons = simpletonConversions flatMap { case SimpletonConversion(_, _, simpletons) => simpletons } val actualSimpletons = wdlValues.simplify @@ -239,17 +298,23 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc // coerceable back into the original type: it should "decompose then reconstruct a map in an object into a coerceable value" in { - val aMap = WomMap(WomMapType(WomStringType, WomArrayType(WomStringType)), Map( - WomString("aa") -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), - WomString("ab") -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) - )) + val aMap = WomMap( + WomMapType(WomStringType, WomArrayType(WomStringType)), + Map( + WomString("aa") -> WomArray(WomArrayType(WomStringType), Seq(WomString("aardvark"), WomString("aaron"))), + WomString("ab") -> WomArray(WomArrayType(WomStringType), Seq(WomString("abacus"), WomString("a bee"))) + ) + ) - val bMap = WomMap(WomMapType(WomStringType, WomArrayType(WomStringType)), Map( - WomString("ba") -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), - WomString("bb") -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) - )) + val bMap = WomMap( + WomMapType(WomStringType, WomArrayType(WomStringType)), + Map( + WomString("ba") -> WomArray(WomArrayType(WomStringType), Seq(WomString("baa"), WomString("battle"))), + WomString("bb") -> WomArray(WomArrayType(WomStringType), Seq(WomString("bbrrrr"), WomString("bb gun"))) + ) + ) - val initial = WomObject(Map("a" -> aMap, "b" -> bMap )) + val initial = WomObject(Map("a" -> aMap, "b" -> bMap)) val map = Map(WomMocks.mockOutputPort("map_in_object") -> initial) @@ -263,9 +328,10 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc WomValueSimpleton("map_in_object:b:ba[0]", WomString("baa")), WomValueSimpleton("map_in_object:b:ba[1]", WomString("battle")), WomValueSimpleton("map_in_object:b:bb[0]", WomString("bbrrrr")), - WomValueSimpleton("map_in_object:b:bb[1]", WomString("bb gun")), + WomValueSimpleton("map_in_object:b:bb[1]", WomString("bb gun")) ), - actualSimpletons) + actualSimpletons + ) // Reconstruct: val outputPort = WomMocks.mockOutputPort(OutputDefinition("map_in_object", initial.womType, IgnoredExpression)) @@ -283,7 +349,8 @@ class WomValueBuilderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc } private def assertSimpletonsEqual(expectedSimpletons: Iterable[WomValueSimpleton], - actualSimpletons: Iterable[WomValueSimpleton]): Unit = { + actualSimpletons: Iterable[WomValueSimpleton] + ): Unit = { // Sanity check, make sure we don't lose anything when we "toSet": actualSimpletons.toSet should contain theSameElementsAs actualSimpletons diff --git a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala index ef1183e9a07..c237ee8c127 100644 --- a/core/src/test/scala/cromwell/util/AkkaTestUtil.scala +++ b/core/src/test/scala/cromwell/util/AkkaTestUtil.scala @@ -39,7 +39,9 @@ object AkkaTestUtil { class DeathTestActor extends Actor { protected def stoppingReceive: Actor.Receive = { case InternalStop => context.stop(self) - case ThrowException => throw new Exception("Don't panic, dear debugger! This was a deliberate exception for the test case.") with NoStackTrace + case ThrowException => + throw new Exception("Don't panic, dear debugger! This was a deliberate exception for the test case.") + with NoStackTrace } override def receive = stoppingReceive orElse Actor.ignoringBehavior } @@ -55,6 +57,6 @@ object AkkaTestUtil { def loggedReceive: Receive - override final def receive: Receive = logMessage orElse loggedReceive + final override def receive: Receive = logMessage orElse loggedReceive } } diff --git a/core/src/test/scala/cromwell/util/EncryptionSpec.scala b/core/src/test/scala/cromwell/util/EncryptionSpec.scala index d1d64f3725d..f9b91107c96 100644 --- a/core/src/test/scala/cromwell/util/EncryptionSpec.scala +++ b/core/src/test/scala/cromwell/util/EncryptionSpec.scala @@ -14,7 +14,8 @@ object EncryptionSpec { """| |Did you install the Java Cryptography Extension (JCE) Unlimited Strength Jurisdiction Policy Files? |http://www.oracle.com/technetwork/java/javase/downloads/jce8-download-2133166.html - |""".stripMargin) + |""".stripMargin + ) } class EncryptionSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { diff --git a/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala b/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala index 875ffd48e3d..214ca6c8454 100644 --- a/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala +++ b/core/src/test/scala/cromwell/util/GracefulShutdownHelperSpec.scala @@ -10,17 +10,17 @@ import org.scalatest.matchers.should.Matchers class GracefulShutdownHelperSpec extends TestKitSuite with AnyFlatSpecLike with Matchers { behavior of "GracefulShutdownHelper" - + it should "send ShutdownCommand to actors, wait for them to shutdown, then shut itself down" in { val testProbeA = TestProbe() val testProbeB = TestProbe() - + val testActor = system.actorOf(Props(new Actor with GracefulShutdownHelper with ActorLogging { - override def receive: Receive = { - case ShutdownCommand => waitForActorsAndShutdown(NonEmptyList.of(testProbeA.ref, testProbeB.ref)) + override def receive: Receive = { case ShutdownCommand => + waitForActorsAndShutdown(NonEmptyList.of(testProbeA.ref, testProbeB.ref)) } })) - + watch(testActor) testActor ! ShutdownCommand @@ -37,7 +37,7 @@ class GracefulShutdownHelperSpec extends TestKitSuite with AnyFlatSpecLike with expectNoMessage() system stop testProbeB.ref - + expectTerminated(testActor) } } diff --git a/core/src/test/scala/cromwell/util/SampleWdl.scala b/core/src/test/scala/cromwell/util/SampleWdl.scala index 6e790ca67be..8180049fc73 100644 --- a/core/src/test/scala/cromwell/util/SampleWdl.scala +++ b/core/src/test/scala/cromwell/util/SampleWdl.scala @@ -2,7 +2,12 @@ package cromwell.util import java.util.UUID import cromwell.core.path.{DefaultPath, DefaultPathBuilder, Path} -import cromwell.core.{WorkflowOptions, WorkflowSourceFilesCollection, WorkflowSourceFilesWithDependenciesZip, WorkflowSourceFilesWithoutImports} +import cromwell.core.{ + WorkflowOptions, + WorkflowSourceFilesCollection, + WorkflowSourceFilesWithDependenciesZip, + WorkflowSourceFilesWithoutImports +} import spray.json._ import wom.core.{ExecutableInputMap, WorkflowJson, WorkflowSource} import wom.values._ @@ -30,7 +35,8 @@ trait SampleWdl extends TestFileUtil { labels: String = "{}", workflowType: Option[String] = Option("WDL"), workflowTypeVersion: Option[String] = None, - workflowOnHold: Boolean = false): WorkflowSourceFilesCollection = { + workflowOnHold: Boolean = false + ): WorkflowSourceFilesCollection = importsZip match { case Some(zip) => WorkflowSourceFilesWithDependenciesZip( @@ -45,7 +51,8 @@ trait SampleWdl extends TestFileUtil { warnings = Vector.empty, workflowOnHold = workflowOnHold, importsZip = zip, - requestedWorkflowId = None) + requestedWorkflowId = None + ) case None => WorkflowSourceFilesWithoutImports( workflowSource = Option(workflowSource(runtime)), @@ -58,9 +65,9 @@ trait SampleWdl extends TestFileUtil { workflowTypeVersion = workflowTypeVersion, warnings = Vector.empty, workflowOnHold = workflowOnHold, - requestedWorkflowId = None) + requestedWorkflowId = None + ) } - } val rawInputs: ExecutableInputMap @@ -84,8 +91,8 @@ trait SampleWdl extends TestFileUtil { def write(x: Any): JsValue = x match { case n: Int => JsNumber(n) case s: String => JsString(s) - case b: Boolean => if(b) JsTrue else JsFalse - case s: Seq[Any] => JsArray(s map {_.toJson} toVector) + case b: Boolean => if (b) JsTrue else JsFalse + case s: Seq[Any] => JsArray(s map { _.toJson } toVector) case a: WomArray => write(a.value) case s: WomString => JsString(s.value) case i: WomInteger => JsNumber(i.value) @@ -111,20 +118,20 @@ object SampleWdl { object HelloWorld extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task hello { - | String addressee - | command { - | echo "Hello $${addressee}!" - | } - | output { - | String salutation = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - |} + |task hello { + | String addressee + | command { + | echo "Hello $${addressee}!" + | } + | output { + | String salutation = read_string(stdout()) + | } + | RUNTIME + |} + | + |workflow wf_hello { + | call hello + |} """.stripMargin.replace("RUNTIME", runtime) val Addressee = "wf_hello.hello.addressee" @@ -157,35 +164,35 @@ object SampleWdl { object EmptyString extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task hello { - | command { - | echo "Hello!" - | } - | output { - | String empty = "" - | } - | RUNTIME - |} - | - |task goodbye { - | String emptyInputString - | command { - | echo "$${emptyInputString}" - | } - | output { - | String empty = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf_hello { - | call hello - | call goodbye {input: emptyInputString=hello.empty } - | output { - | hello.empty - | goodbye.empty - | } - |} + |task hello { + | command { + | echo "Hello!" + | } + | output { + | String empty = "" + | } + | RUNTIME + |} + | + |task goodbye { + | String emptyInputString + | command { + | echo "$${emptyInputString}" + | } + | output { + | String empty = read_string(stdout()) + | } + | RUNTIME + |} + | + |workflow wf_hello { + | call hello + | call goodbye {input: emptyInputString=hello.empty } + | output { + | hello.empty + | goodbye.empty + | } + |} """.stripMargin.replace("RUNTIME", runtime) val rawInputs = Map.empty[String, Any] @@ -196,32 +203,31 @@ object SampleWdl { } object CoercionNotDefined extends SampleWdl { - override def workflowSource(runtime: String = ""): WorkflowSource = { + override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task summary { - | String bfile - | command { - | ~/plink --bfile $${bfile} --missing --hardy --out foo --allow-no-sex - | } - | output { - | File hwe = "foo.hwe" - | File log = "foo.log" - | File imiss = "foo.imiss" - | File lmiss = "foo.lmiss" - | } - | meta { - | author: "Jackie Goldstein" - | email: "jigold@broadinstitute.org" - | } - |} - | - |workflow test1 { - | call summary { - | input: bfile = bfile - | } - |} + |task summary { + | String bfile + | command { + | ~/plink --bfile $${bfile} --missing --hardy --out foo --allow-no-sex + | } + | output { + | File hwe = "foo.hwe" + | File log = "foo.log" + | File imiss = "foo.imiss" + | File lmiss = "foo.lmiss" + | } + | meta { + | author: "Jackie Goldstein" + | email: "jigold@broadinstitute.org" + | } + |} + | + |workflow test1 { + | call summary { + | input: bfile = bfile + | } + |} """.stripMargin - } override val rawInputs: ExecutableInputMap = Map("test1.bfile" -> "data/example1") } @@ -281,67 +287,66 @@ object SampleWdl { withPlaceholders.stripMargin.replace(outputSectionPlaceholder, outputsSection) } - val PatternKey ="three_step.cgrep.pattern" + val PatternKey = "three_step.cgrep.pattern" override lazy val rawInputs = Map(PatternKey -> "...") } object ThreeStep extends ThreeStepTemplate object ThreeStepWithOutputsSection extends ThreeStepTemplate { - override def workflowSource(runtime: String = ""): WorkflowJson = sourceString(outputsSection = - """ - |output { - | cgrep.count - | wc.count - |} + override def workflowSource(runtime: String = ""): WorkflowJson = sourceString(outputsSection = """ + |output { + | cgrep.count + | wc.count + |} """.stripMargin).replaceAll("RUNTIME", runtime) } object DeclarationsWorkflow extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s""" - |task cat { - | File file - | String? flags - | String? flags2 # This should be a workflow input - | command { - | cat $${flags} $${flags2} $${file} - | } - | output { - | File procs = stdout() - | } - |} - | - |task cgrep { - | String str_decl - | String pattern - | File in_file - | command { - | grep '$${pattern}' $${in_file} | wc -l - | } - | output { - | Int count = read_int(stdout()) - | String str = str_decl - | } - |} - | - |workflow two_step { - | String flags_suffix - | String flags = "-" + flags_suffix - | String static_string = "foobarbaz" - | call cat { - | input: flags = flags - | } - | call cgrep { - | input: in_file = cat.procs - | } - |} + |task cat { + | File file + | String? flags + | String? flags2 # This should be a workflow input + | command { + | cat $${flags} $${flags2} $${file} + | } + | output { + | File procs = stdout() + | } + |} + | + |task cgrep { + | String str_decl + | String pattern + | File in_file + | command { + | grep '$${pattern}' $${in_file} | wc -l + | } + | output { + | Int count = read_int(stdout()) + | String str = str_decl + | } + |} + | + |workflow two_step { + | String flags_suffix + | String flags = "-" + flags_suffix + | String static_string = "foobarbaz" + | call cat { + | input: flags = flags + | } + | call cgrep { + | input: in_file = cat.procs + | } + |} """.stripMargin private val fileContents = s"""first line - |second line - |third line + |second line + |third line """.stripMargin override val rawInputs: ExecutableInputMap = Map( @@ -446,94 +451,94 @@ object SampleWdl { object ArrayIO extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |task serialize { - | Array[String] strs - | command { - | cat $${write_lines(strs)} - | } - | output { - | String contents = read_string(stdout()) - | } - | RUNTIME - |} - | - |workflow wf { - | Array[String] strings = ["str1", "str2", "str3"] - | call serialize { - | input: strs = strings - | } - |} + |task serialize { + | Array[String] strs + | command { + | cat $${write_lines(strs)} + | } + | output { + | String contents = read_string(stdout()) + | } + | RUNTIME + |} + | + |workflow wf { + | Array[String] strings = ["str1", "str2", "str3"] + | call serialize { + | input: strs = strings + | } + |} """.stripMargin.replace("RUNTIME", runtime) override val rawInputs: Map[String, Any] = Map.empty } class ScatterWdl extends SampleWdl { val tasks: String = s"""task A { - | command { - | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nruchi" - | } - | RUNTIME - | output { - | Array[String] A_out = read_lines(stdout()) - | } - |} - | - |task B { - | String B_in - | command { - | python -c "print(len('$${B_in}'))" - | } - | RUNTIME - | output { - | Int B_out = read_int(stdout()) - | } - |} - | - |task C { - | Int C_in - | command { - | python -c "print($${C_in}*100)" - | } - | RUNTIME - | output { - | Int C_out = read_int(stdout()) - | } - |} - | - |task D { - | Array[Int] D_in - | command { - | python -c "print($${sep = '+' D_in})" - | } - | RUNTIME - | output { - | Int D_out = read_int(stdout()) - | } - |} - | - |task E { - | command { - | python -c "print(9)" - | } - | RUNTIME - | output { - | Int E_out = read_int(stdout()) - | } - |} + | command { + | echo -n -e "jeff\nchris\nmiguel\nthibault\nkhalid\nruchi" + | } + | RUNTIME + | output { + | Array[String] A_out = read_lines(stdout()) + | } + |} + | + |task B { + | String B_in + | command { + | python -c "print(len('$${B_in}'))" + | } + | RUNTIME + | output { + | Int B_out = read_int(stdout()) + | } + |} + | + |task C { + | Int C_in + | command { + | python -c "print($${C_in}*100)" + | } + | RUNTIME + | output { + | Int C_out = read_int(stdout()) + | } + |} + | + |task D { + | Array[Int] D_in + | command { + | python -c "print($${sep = '+' D_in})" + | } + | RUNTIME + | output { + | Int D_out = read_int(stdout()) + | } + |} + | + |task E { + | command { + | python -c "print(9)" + | } + | RUNTIME + | output { + | Int E_out = read_int(stdout()) + | } + |} """.stripMargin override def workflowSource(runtime: String = ""): WorkflowSource = s"""$tasks - | - |workflow w { - | call A - | scatter (item in A.A_out) { - | call B {input: B_in = item} - | call C {input: C_in = B.B_out} - | call E - | } - | call D {input: D_in = B.B_out} - |} + | + |workflow w { + | call A + | scatter (item in A.A_out) { + | call B {input: B_in = item} + | call C {input: C_in = B.B_out} + | call E + | } + | call D {input: D_in = B.B_out} + |} """.stripMargin.replace("RUNTIME", runtime) override lazy val rawInputs = Map.empty[String, String] @@ -542,21 +547,21 @@ object SampleWdl { object SimpleScatterWdl extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task echo_int { - | Int int - | command {echo $${int}} - | output {Int out = read_int(stdout())} - | RUNTIME_PLACEHOLDER - |} - | - |workflow scatter0 { - | Array[Int] ints = [1,2,3,4,5] - | call echo_int as outside_scatter {input: int = 8000} - | scatter(i in ints) { - | call echo_int as inside_scatter { - | input: int = i - | } - | } - |} + | Int int + | command {echo $${int}} + | output {Int out = read_int(stdout())} + | RUNTIME_PLACEHOLDER + |} + | + |workflow scatter0 { + | Array[Int] ints = [1,2,3,4,5] + | call echo_int as outside_scatter {input: int = 8000} + | scatter(i in ints) { + | call echo_int as inside_scatter { + | input: int = i + | } + | } + |} """.stripMargin.replace("RUNTIME_PLACEHOLDER", runtime) override lazy val rawInputs = Map.empty[String, String] @@ -565,108 +570,108 @@ object SampleWdl { object SimpleScatterWdlWithOutputs extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task echo_int { - | Int int - | command {echo $${int}} - | output {Int out = read_int(stdout())} - |} - | - |workflow scatter0 { - | Array[Int] ints = [1,2,3,4,5] - | call echo_int as outside_scatter {input: int = 8000} - | scatter(i in ints) { - | call echo_int as inside_scatter { - | input: int = i - | } - | } - | output { - | inside_scatter.* - | } - |} + | Int int + | command {echo $${int}} + | output {Int out = read_int(stdout())} + |} + | + |workflow scatter0 { + | Array[Int] ints = [1,2,3,4,5] + | call echo_int as outside_scatter {input: int = 8000} + | scatter(i in ints) { + | call echo_int as inside_scatter { + | input: int = i + | } + | } + | output { + | inside_scatter.* + | } + |} """.stripMargin override lazy val rawInputs = Map.empty[String, String] } case class PrepareScatterGatherWdl(salt: String = UUID.randomUUID().toString) extends SampleWdl { - override def workflowSource(runtime: String = ""): WorkflowSource = { + override def workflowSource(runtime: String = ""): WorkflowSource = s""" - |# - |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with - |# the word-count for the fil - |# - |# splits each line into a file with the name temp_?? (shuffle) - |task do_prepare { - | File input_file - | command { - | split -l 1 $${input_file} temp_ && ls -1 temp_?? > files.list - | } - | output { - | Array[File] split_files = read_lines("files.list") - | } - | RUNTIME - |} - |# count the number of words in the input file, writing the count to an output file overkill in this case, but simulates a real scatter-gather that would just return an Int (map) - |task do_scatter { - | String salt - | File input_file - | command { - | # $${salt} - | wc -w $${input_file} > output.txt - | } - | output { - | File count_file = "output.txt" - | } - | RUNTIME - |} - |# aggregate the results back together (reduce) - |task do_gather { - | Array[File] input_files - | command <<< - | cat $${sep = ' ' input_files} | awk '{s+=$$1} END {print s}' - | >>> - | output { - | Int sum = read_int(stdout()) - | } - | RUNTIME - |} - |workflow sc_test { - | call do_prepare - | scatter(f in do_prepare.split_files) { - | call do_scatter { - | input: input_file = f - | } - | } - | call do_gather { - | input: input_files = do_scatter.count_file - | } - |} + |# + |# Goal here is to split up the input file into files of 1 line each (in the prepare) then in parallel call wc -w on each newly created file and count the words into another file then in the gather, sum the results of each parallel call to come up with + |# the word-count for the fil + |# + |# splits each line into a file with the name temp_?? (shuffle) + |task do_prepare { + | File input_file + | command { + | split -l 1 $${input_file} temp_ && ls -1 temp_?? > files.list + | } + | output { + | Array[File] split_files = read_lines("files.list") + | } + | RUNTIME + |} + |# count the number of words in the input file, writing the count to an output file overkill in this case, but simulates a real scatter-gather that would just return an Int (map) + |task do_scatter { + | String salt + | File input_file + | command { + | # $${salt} + | wc -w $${input_file} > output.txt + | } + | output { + | File count_file = "output.txt" + | } + | RUNTIME + |} + |# aggregate the results back together (reduce) + |task do_gather { + | Array[File] input_files + | command <<< + | cat $${sep = ' ' input_files} | awk '{s+=$$1} END {print s}' + | >>> + | output { + | Int sum = read_int(stdout()) + | } + | RUNTIME + |} + |workflow sc_test { + | call do_prepare + | scatter(f in do_prepare.split_files) { + | call do_scatter { + | input: input_file = f + | } + | } + | call do_gather { + | input: input_files = do_scatter.count_file + | } + |} """.stripMargin.replace("RUNTIME", runtime) - } val contents: String = - """|the - |total number - |of words in this - |text file is 11 - |""".stripMargin + """|the + |total number + |of words in this + |text file is 11 + |""".stripMargin override lazy val rawInputs = Map( "sc_test.do_prepare.input_file" -> createCannedFile("scatter", contents).pathAsString, - "sc_test.do_scatter.salt" -> salt) + "sc_test.do_scatter.salt" -> salt + ) } object FileClobber extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task read_line { - | File in - | command { cat $${in} } - | output { String out = read_string(stdout()) } - |} - | - |workflow two { - | call read_line as x - | call read_line as y - |} + | File in + | command { cat $${in} } + | output { String out = read_string(stdout()) } + |} + | + |workflow two { + | call read_line as x + | call read_line as y + |} """.stripMargin val tempDir1: DefaultPath = DefaultPathBuilder.createTempDirectory("FileClobber1") @@ -683,26 +688,26 @@ object SampleWdl { object FilePassingWorkflow extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s"""task a { - | File in - | String out_name = "out" - | - | command { - | cat $${in} > $${out_name} - | } - | RUNTIME - | output { - | File out = "out" - | File out_interpolation = "$${out_name}" - | String contents = read_string("$${out_name}") - | } - |} - | - |workflow file_passing { - | File f - | - | call a {input: in = f} - | call a as b {input: in = a.out} - |} + | File in + | String out_name = "out" + | + | command { + | cat $${in} > $${out_name} + | } + | RUNTIME + | output { + | File out = "out" + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") + | } + |} + | + |workflow file_passing { + | File f + | + | call a {input: in = f} + | call a as b {input: in = a.out} + |} """.stripMargin.replace("RUNTIME", runtime) private val fileContents = s"foo bar baz" @@ -723,30 +728,30 @@ object SampleWdl { case class CallCachingWorkflow(salt: String) extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s"""task a { - | File in - | String out_name = "out" - | String salt - | - | command { - | # $${salt} - | echo "Something" - | cat $${in} > $${out_name} - | } - | RUNTIME - | output { - | File out = "out" - | File out_interpolation = "$${out_name}" - | String contents = read_string("$${out_name}") - | Array[String] stdoutContent = read_lines(stdout()) - | } - |} - | - |workflow file_passing { - | File f - | - | call a {input: in = f} - | call a as b {input: in = a.out} - |} + | File in + | String out_name = "out" + | String salt + | + | command { + | # $${salt} + | echo "Something" + | cat $${in} > $${out_name} + | } + | RUNTIME + | output { + | File out = "out" + | File out_interpolation = "$${out_name}" + | String contents = read_string("$${out_name}") + | Array[String] stdoutContent = read_lines(stdout()) + | } + |} + | + |workflow file_passing { + | File f + | + | call a {input: in = f} + | call a as b {input: in = a.out} + |} """.stripMargin.replace("RUNTIME", runtime) private val fileContents = s"foo bar baz" @@ -761,29 +766,29 @@ object SampleWdl { object CallCachingHashingWdl extends SampleWdl { override def workflowSource(runtime: String): WorkflowSource = s"""task t { - | Int a - | Float b - | String c - | File d - | - | command { - | echo "$${a}" > a - | echo "$${b}" > b - | echo "$${c}" > c - | cat $${d} > d - | } - | output { - | Int w = read_int("a") + 2 - | Float x = read_float("b") - | String y = read_string("c") - | File z = "d" - | } - | RUNTIME - |} - | - |workflow w { - | call t - |} + | Int a + | Float b + | String c + | File d + | + | command { + | echo "$${a}" > a + | echo "$${b}" > b + | echo "$${c}" > c + | cat $${d} > d + | } + | output { + | Int w = read_int("a") + 2 + | Float x = read_float("b") + | String y = read_string("c") + | File z = "d" + | } + | RUNTIME + |} + | + |workflow w { + | call t + |} """.stripMargin.replace("RUNTIME", runtime) val tempDir: DefaultPath = DefaultPathBuilder.createTempDirectory("CallCachingHashingWdl") @@ -799,26 +804,26 @@ object SampleWdl { object ExpressionsInInputs extends SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s"""task echo { - | String inString - | command { - | echo $${inString} - | } - | - | output { - | String outString = read_string(stdout()) - | } - |} - | - |workflow wf { - | String a1 - | String a2 - | call echo { - | input: inString = a1 + " " + a2 - | } - | call echo as echo2 { - | input: inString = a1 + " " + echo.outString + " " + a2 - | } - |} + | String inString + | command { + | echo $${inString} + | } + | + | output { + | String outString = read_string(stdout()) + | } + |} + | + |workflow wf { + | String a1 + | String a2 + | call echo { + | input: inString = a1 + " " + a2 + | } + | call echo as echo2 { + | input: inString = a1 + " " + echo.outString + " " + a2 + | } + |} """.stripMargin override val rawInputs = Map( "wf.a1" -> WomString("hello"), @@ -830,61 +835,61 @@ object SampleWdl { override def workflowSource(runtime: String = ""): WorkflowSource = s""" task shouldCompleteFast { - | Int a - | command { - | echo "The number was: $${a}" - | } - | output { - | Int echo = a - | } - |} - | - |task shouldCompleteSlow { - | Int a - | command { - | echo "The number was: $${a}" - | # More than 1 so this should finish second - | sleep 2 - | } - | output { - | Int echo = a - | } - |} - | - |task failMeSlowly { - | Int a - | command { - | echo "The number was: $${a}" - | # Less than 2 so this should finish first - | sleep 1 - | ./NOOOOOO - | } - | output { - | Int echo = a - | } - |} - | - |task shouldNeverRun { - | Int a - | Int b - | command { - | echo "You can't fight in here - this is the war room $${a + b}" - | } - | output { - | Int echo = a - | } - |} - | - |workflow wf { - | call shouldCompleteFast as A { input: a = 5 } - | call shouldCompleteFast as B { input: a = 5 } - | - | call failMeSlowly as ohNOOOOOOOO { input: a = A.echo } - | call shouldCompleteSlow as C { input: a = B.echo } - | - | call shouldNeverRun as D { input: a = ohNOOOOOOOO.echo, b = C.echo } - | call shouldCompleteSlow as E { input: a = C.echo } - |} + | Int a + | command { + | echo "The number was: $${a}" + | } + | output { + | Int echo = a + | } + |} + | + |task shouldCompleteSlow { + | Int a + | command { + | echo "The number was: $${a}" + | # More than 1 so this should finish second + | sleep 2 + | } + | output { + | Int echo = a + | } + |} + | + |task failMeSlowly { + | Int a + | command { + | echo "The number was: $${a}" + | # Less than 2 so this should finish first + | sleep 1 + | ./NOOOOOO + | } + | output { + | Int echo = a + | } + |} + | + |task shouldNeverRun { + | Int a + | Int b + | command { + | echo "You can't fight in here - this is the war room $${a + b}" + | } + | output { + | Int echo = a + | } + |} + | + |workflow wf { + | call shouldCompleteFast as A { input: a = 5 } + | call shouldCompleteFast as B { input: a = 5 } + | + | call failMeSlowly as ohNOOOOOOOO { input: a = A.echo } + | call shouldCompleteSlow as C { input: a = B.echo } + | + | call shouldNeverRun as D { input: a = ohNOOOOOOOO.echo, b = C.echo } + | call shouldCompleteSlow as E { input: a = C.echo } + |} """.stripMargin val rawInputs = Map( diff --git a/core/src/test/scala/cromwell/util/TestFileUtil.scala b/core/src/test/scala/cromwell/util/TestFileUtil.scala index dbdad8a47ed..f13fb57891b 100644 --- a/core/src/test/scala/cromwell/util/TestFileUtil.scala +++ b/core/src/test/scala/cromwell/util/TestFileUtil.scala @@ -17,9 +17,8 @@ trait TestFileUtil { tempFile.write(contents) } - def createFile(name: String, dir: Path, contents: String): Path = { + def createFile(name: String, dir: Path, contents: String): Path = dir.createPermissionedDirectories()./(name).write(contents) - } } trait HashUtil extends TestFileUtil { @@ -36,6 +35,7 @@ trait HashUtil extends TestFileUtil { object ErrorOrUtil { implicit class EnhancedErrorOr[A](val value: ErrorOr[A]) extends AnyVal { + /** Extract a value from an `ErrorOr` box if the box is `Valid`, throw an exception if the box is `Invalid`. * For test code only. */ def get: A = value match { diff --git a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala index 0764bdd9826..b15cdcebec7 100644 --- a/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala +++ b/core/src/test/scala/cromwell/util/TryWithResourceSpec.scala @@ -11,31 +11,34 @@ class TryWithResourceSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matc behavior of "tryWithResource" it should "catch instantiation errors" in { - val triedMyBest = tryWithResource(() => if (1 == 1) throw InstantiationException else null) { _ => 5 } + val triedMyBest = tryWithResource(() => if (1 == 1) throw InstantiationException else null)(_ => 5) triedMyBest should be(Failure(InstantiationException)) } it should "close the closeable" in { val myCloseable = new MyCloseable - val triedMyBest = tryWithResource(() => myCloseable) { _.value } // Nothing special about 5... Just need to return something! + val triedMyBest = tryWithResource(() => myCloseable) { + _.value + } // Nothing special about 5... Just need to return something! triedMyBest should be(Success(5)) myCloseable.isClosed should be(true) } it should "catch errors and still close the closeable" in { val myCloseable = new MyCloseable - val triedMyBest = tryWithResource(() => myCloseable) { _.badValue } + val triedMyBest = tryWithResource(() => myCloseable)(_.badValue) triedMyBest should be(Failure(ReadValueException)) myCloseable.isClosed should be(true) } it should "be robust to failures in close methods" in { val myCloseable = new FailingCloseable - val triedMyBest = tryWithResource(() => myCloseable) { _.value } + val triedMyBest = tryWithResource(() => myCloseable)(_.value) triedMyBest should be(Failure(CloseCloseableException)) - val triedMyBest2 = tryWithResource(() => myCloseable) { _.badValue } + val triedMyBest2 = tryWithResource(() => myCloseable)(_.badValue) triedMyBest2 match { - case Failure(ReadValueException) => ReadValueException.getSuppressed.headOption should be(Some(CloseCloseableException)) + case Failure(ReadValueException) => + ReadValueException.getSuppressed.headOption should be(Some(CloseCloseableException)) case x => fail(s"$x was not equal to $ReadValueException") } } @@ -47,9 +50,8 @@ class MyCloseable extends AutoCloseable { val value = if (isClosed) throw ReadValueException else 5 // Ensures we aren't closed when .value is called def badValue = throw ReadValueException - override def close() = { + override def close() = isClosed = true - } } class FailingCloseable extends MyCloseable { diff --git a/core/src/test/scala/cromwell/util/WomMocks.scala b/core/src/test/scala/cromwell/util/WomMocks.scala index 2dc75385658..07111be17ce 100644 --- a/core/src/test/scala/cromwell/util/WomMocks.scala +++ b/core/src/test/scala/cromwell/util/WomMocks.scala @@ -6,44 +6,71 @@ import wom.RuntimeAttributes import wom.callable.Callable.OutputDefinition import wom.callable.{CallableTaskDefinition, CommandTaskDefinition, WorkflowDefinition} import wom.graph.GraphNodePort.{GraphNodeOutputPort, OutputPort} -import wom.graph.{Graph, CommandCallNode, WomIdentifier, WorkflowCallNode} +import wom.graph.{CommandCallNode, Graph, WomIdentifier, WorkflowCallNode} import wom.types.{WomStringType, WomType} import wom.values.WomValue object WomMocks { - val EmptyTaskDefinition = CallableTaskDefinition("emptyTask", Function.const(List.empty.validNel), RuntimeAttributes(Map.empty), - Map.empty, Map.empty, List.empty, List.empty, Set.empty, Map.empty, sourceLocation = None) + val EmptyTaskDefinition = CallableTaskDefinition( + "emptyTask", + Function.const(List.empty.validNel), + RuntimeAttributes(Map.empty), + Map.empty, + Map.empty, + List.empty, + List.empty, + Set.empty, + Map.empty, + sourceLocation = None + ) val EmptyWorkflowDefinition = mockWorkflowDefinition("emptyWorkflow") - def mockTaskCall(identifier: WomIdentifier, definition: CommandTaskDefinition = EmptyTaskDefinition) = { - CommandCallNode(identifier, definition, Set.empty, List.empty, Set.empty, (_, localName) => WomIdentifier(localName = localName), None) - } + def mockTaskCall(identifier: WomIdentifier, definition: CommandTaskDefinition = EmptyTaskDefinition) = + CommandCallNode(identifier, + definition, + Set.empty, + List.empty, + Set.empty, + (_, localName) => WomIdentifier(localName = localName), + None + ) - def mockWorkflowCall(identifier: WomIdentifier, definition: WorkflowDefinition = EmptyWorkflowDefinition) = { - WorkflowCallNode(identifier, definition, Set.empty, List.empty, Set.empty, (_, localName) => identifier.combine(localName), None) - } + def mockWorkflowCall(identifier: WomIdentifier, definition: WorkflowDefinition = EmptyWorkflowDefinition) = + WorkflowCallNode(identifier, + definition, + Set.empty, + List.empty, + Set.empty, + (_, localName) => identifier.combine(localName), + None + ) - def mockWorkflowDefinition(name: String) = { + def mockWorkflowDefinition(name: String) = WorkflowDefinition(name, Graph(Set.empty), Map.empty, Map.empty, None) - } - def mockTaskDefinition(name: String) = { - CallableTaskDefinition(name, Function.const(List.empty.validNel), RuntimeAttributes(Map.empty), - Map.empty, Map.empty, List.empty, List.empty, Set.empty, Map.empty, sourceLocation = None) - } + def mockTaskDefinition(name: String) = + CallableTaskDefinition( + name, + Function.const(List.empty.validNel), + RuntimeAttributes(Map.empty), + Map.empty, + Map.empty, + List.empty, + List.empty, + Set.empty, + Map.empty, + sourceLocation = None + ) - def mockOutputPort(name: String, womType: WomType = WomStringType): OutputPort = { + def mockOutputPort(name: String, womType: WomType = WomStringType): OutputPort = GraphNodeOutputPort(WomIdentifier(name, name), womType, null) - } - def mockOutputPort(outputDefinition: OutputDefinition): OutputPort = { + def mockOutputPort(outputDefinition: OutputDefinition): OutputPort = GraphNodeOutputPort(WomIdentifier(outputDefinition.name, outputDefinition.name), outputDefinition.womType, null) - } - def mockOutputExpectations(outputs: Map[String, WomValue]): CallOutputs = { - CallOutputs(outputs.map { - case (key, value) => WomMocks.mockOutputPort(key, value.womType) -> value + def mockOutputExpectations(outputs: Map[String, WomValue]): CallOutputs = + CallOutputs(outputs.map { case (key, value) => + WomMocks.mockOutputPort(key, value.womType) -> value }) - } } diff --git a/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala b/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala index 1c4cc15dfdd..69739d9868f 100644 --- a/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala +++ b/core/src/test/scala/cromwell/util/WomValueJsonFormatterSpec.scala @@ -4,7 +4,7 @@ import common.assertion.CromwellTimeoutSpec import cromwell.util.JsonFormatting.WomValueJsonFormatter.WomValueJsonFormat import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers -import spray.json.{JsObject, enrichString} +import spray.json.{enrichString, JsObject} import wom.types._ import wom.values._ @@ -15,7 +15,7 @@ class WomValueJsonFormatterSpec extends AnyFlatSpec with CromwellTimeoutSpec wit it should "write WdlPair to left/right structured JsObject" in { val left = "sanders" val right = Vector("rubio", "carson", "cruz") - val wdlPair = WomPair(WomString(left), WomArray(WomArrayType(WomStringType), right.map { WomString(_) })) + val wdlPair = WomPair(WomString(left), WomArray(WomArrayType(WomStringType), right.map(WomString(_)))) val ExpectedJson: JsObject = """|{ | "left": "sanders", diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala index df0f7c2eba6..5aba6092339 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/CommandLineParser.scala @@ -4,7 +4,6 @@ import common.util.VersionUtil import drs.localizer.CommandLineParser.AccessTokenStrategy._ import drs.localizer.CommandLineParser.Usage - class CommandLineParser extends scopt.OptionParser[CommandLineArguments](Usage) { lazy val localizerVersion: String = VersionUtil.getVersion("cromwell-drs-localizer") @@ -14,43 +13,63 @@ class CommandLineParser extends scopt.OptionParser[CommandLineArguments](Usage) head("cromwell-drs-localizer", localizerVersion) - arg[String]("drs-object-id").text("DRS object ID").required(). - action((s, c) => - c.copy(drsObject = Option(s))) - arg[String]("container-path").text("Container path").required(). - action((s, c) => - c.copy(containerPath = Option(s))) - arg[String]("requester-pays-project").text("Requester pays project").optional(). - action((s, c) => - c.copy(googleRequesterPaysProject = Option(s))) - opt[String]('t', "access-token-strategy").text(s"Access token strategy, must be one of '$Azure' or '$Google' (default '$Google')"). - action((s, c) => - c.copy(accessTokenStrategy = Option(s.toLowerCase()))) - opt[String]('v', "vault-name").text("Azure vault name"). - action((s, c) => - c.copy(azureVaultName = Option(s))) - opt[String]('s', "secret-name").text("Azure secret name"). - action((s, c) => - c.copy(azureSecretName = Option(s))) - opt[String]('i', "identity-client-id").text("Azure identity client id"). - action((s, c) => - c.copy(azureIdentityClientId = Option(s))) + arg[String]("drs-object-id").text("DRS object ID").optional().action((s, c) => c.copy(drsObject = Option(s))) + arg[String]("container-path").text("Container path").optional().action((s, c) => c.copy(containerPath = Option(s))) + arg[String]("requester-pays-project") + .text(s"Requester pays project (only valid with '$Google' auth strategy)") + .optional() + .action((s, c) => c.copy(googleRequesterPaysProject = Option(s))) + opt[String]('m', "manifest-path") + .text("File path of manifest containing multiple files to localize") + .action((s, c) => c.copy(manifestPath = Option(s))) + opt[String]('r', "requester-pays-project") + .text(s"Requester pays project (only valid with '$Google' auth strategy)") + .optional() + .action { (s, c) => + c.copy( + googleRequesterPaysProject = Option(s), + googleRequesterPaysProjectConflict = c.googleRequesterPaysProject.exists(_ != s) + ) + } + opt[String]('t', "access-token-strategy") + .text(s"Access token strategy, must be one of '$Azure' or '$Google' (default '$Google')") + .action((s, c) => c.copy(accessTokenStrategy = Option(s.toLowerCase()))) + opt[String]('i', "identity-client-id") + .text("Azure identity client id") + .action((s, c) => c.copy(azureIdentityClientId = Option(s))) + checkConfig(c => + if (c.googleRequesterPaysProjectConflict) + failure("Requester pays project differs between positional argument and option flag") + else success + ) + checkConfig(c => + if (c.googleRequesterPaysProjectConflict) + failure("Requester pays project differs between positional argument and option flag") + else success + ) checkConfig(c => c.accessTokenStrategy match { case Some(Azure) if c.googleRequesterPaysProject.nonEmpty => Left(s"Requester pays project is only valid with access token strategy '$Google'") - case Some(Azure) if List(c.azureVaultName, c.azureSecretName).exists(_.isEmpty) => - Left(s"Both vault name and secret name must be specified for access token strategy $Azure") + case Some(Google) if c.azureIdentityClientId.nonEmpty => + Left(s"Identity client id is only valid with access token strategy '$Azure'") case Some(Azure) => Right(()) - case Some(Google) if List(c.azureSecretName, c.azureVaultName, c.azureIdentityClientId).forall(_.isEmpty) => Right(()) - case Some(Google) => Left(s"One or more specified options are only valid with access token strategy '$Azure'") + case Some(Google) => Right(()) case Some(huh) => Left(s"Unrecognized access token strategy '$huh'") case None => Left("Programmer error, access token strategy should not be None") } ) + checkConfig(c => + (c.drsObject, c.containerPath, c.manifestPath) match { + case (Some(_), Some(_), None) => Right(()) + case (None, None, Some(_)) => Right(()) + case _ => Left("Must provide either DRS path and container path, OR manifest file (-m).") + } + ) } object CommandLineParser { + /** * These access token strategies are named simplistically as there is currently only one access token strategy being * used for each of these cloud vendors. But it is certainly possible that multiple strategies could come into use @@ -64,9 +83,12 @@ object CommandLineParser { val Usage = s""" Usage: - java -jar /path/to/localizer.jar [options] drs://provider/object /local/path/to/file.txt [requester pays project] - Note that the optional argument is only valid with access token strategy 'Google'. + Can be run to localize a single file with DRS id and local container path provided in args: + java -jar /path/to/localizer.jar [options] drs://provider/object /local/path/to/file.txt + + Can also be used to localize multiple files in one invocation with manifest file provided in args: + java -jar /path/to/localizer.jar [options] -m /local/path/to/manifest/file """ } @@ -75,6 +97,7 @@ case class CommandLineArguments(accessTokenStrategy: Option[String] = Option(Goo drsObject: Option[String] = None, containerPath: Option[String] = None, googleRequesterPaysProject: Option[String] = None, - azureVaultName: Option[String] = None, - azureSecretName: Option[String] = None, - azureIdentityClientId: Option[String] = None) + azureIdentityClientId: Option[String] = None, + manifestPath: Option[String] = None, + googleRequesterPaysProjectConflict: Boolean = false +) diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala deleted file mode 100644 index ac27367b932..00000000000 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerDrsPathResolver.scala +++ /dev/null @@ -1,10 +0,0 @@ -package drs.localizer - -import cloud.nio.impl.drs.{DrsConfig, DrsPathResolver} -import common.validation.ErrorOr.ErrorOr -import drs.localizer.accesstokens.AccessTokenStrategy - - -class DrsLocalizerDrsPathResolver(drsConfig: DrsConfig, accessTokenStrategy: AccessTokenStrategy) extends DrsPathResolver(drsConfig) { - override def getAccessToken: ErrorOr[String] = accessTokenStrategy.getAccessToken() -} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala index c7ecdd3f3e4..64f36239db5 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/DrsLocalizerMain.scala @@ -2,18 +2,25 @@ package drs.localizer import cats.data.NonEmptyList import cats.effect.{ExitCode, IO, IOApp} +import cats.implicits.toTraverseOps import cloud.nio.impl.drs.DrsPathResolver.{FatalRetryDisposition, RegularRetryDisposition} -import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsPathResolver, MarthaField} +import cloud.nio.impl.drs._ import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff} import com.typesafe.scalalogging.StrictLogging import drs.localizer.CommandLineParser.AccessTokenStrategy.{Azure, Google} -import drs.localizer.accesstokens.{AccessTokenStrategy, AzureB2CAccessTokenStrategy, GoogleAccessTokenStrategy} -import drs.localizer.downloaders.AccessUrlDownloader.Hashes +import drs.localizer.DrsLocalizerMain.{defaultNumRetries, toValidatedUriType} import drs.localizer.downloaders._ +import org.apache.commons.csv.{CSVFormat, CSVParser} +import java.io.File +import java.nio.charset.Charset import scala.concurrent.duration._ +import scala.jdk.CollectionConverters._ import scala.language.postfixOps +import drs.localizer.URIType.URIType +case class UnresolvedDrsUrl(drsUrl: String, downloadDestinationPath: String) +case class ResolvedDrsUrl(drsResponse: DrsResolverResponse, downloadDestinationPath: String, uriType: URIType) object DrsLocalizerMain extends IOApp with StrictLogging { override def run(args: List[String]): IO[ExitCode] = { @@ -24,8 +31,8 @@ object DrsLocalizerMain extends IOApp with StrictLogging { val localize: Option[IO[ExitCode]] = for { pa <- parsedArgs run <- pa.accessTokenStrategy.collect { - case Azure => runLocalizer(pa, AzureB2CAccessTokenStrategy(pa)) - case Google => runLocalizer(pa, GoogleAccessTokenStrategy) + case Azure => runLocalizer(pa, AzureDrsCredentials(pa.azureIdentityClientId)) + case Google => runLocalizer(pa, GoogleAppDefaultTokenStrategy) } } yield run @@ -34,15 +41,21 @@ object DrsLocalizerMain extends IOApp with StrictLogging { def buildParser(): scopt.OptionParser[CommandLineArguments] = new CommandLineParser() - val defaultBackoff: CloudNioBackoff = CloudNioSimpleExponentialBackoff( - initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2) + // Default retry parameters for resolving a DRS url + val defaultNumRetries: Int = 5 + val defaultBackoff: CloudNioBackoff = + CloudNioSimpleExponentialBackoff(initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2) val defaultDownloaderFactory: DownloaderFactory = new DownloaderFactory { - override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = - IO.pure(AccessUrlDownloader(accessUrl, downloadLoc, hashes)) - - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = - IO.pure(GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption)) + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = + GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption) + + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = + BulkAccessUrlDownloader(urlsToDownload) } private def printUsage: IO[ExitCode] = { @@ -50,96 +63,189 @@ object DrsLocalizerMain extends IOApp with StrictLogging { IO.pure(ExitCode.Error) } - def runLocalizer(commandLineArguments: CommandLineArguments, accessTokenStrategy: AccessTokenStrategy): IO[ExitCode] = { - val drsObject = commandLineArguments.drsObject.get - val containerPath = commandLineArguments.containerPath.get - new DrsLocalizerMain(drsObject, containerPath, accessTokenStrategy, commandLineArguments.googleRequesterPaysProject). - resolveAndDownloadWithRetries(downloadRetries = 3, checksumRetries = 1, defaultDownloaderFactory, Option(defaultBackoff)).map(_.exitCode) + /** + * Helper function to read a CSV file as pairs of drsURL -> local download destination. + * @param csvManifestPath Path to a CSV file where each row is something like: drs://asdf.ghj, path/to/my/directory + */ + def loadCSVManifest(csvManifestPath: String): IO[List[UnresolvedDrsUrl]] = + IO { + val openFile = new File(csvManifestPath) + val csvParser = CSVParser.parse(openFile, Charset.defaultCharset(), CSVFormat.DEFAULT) + try + csvParser.getRecords.asScala.map(record => UnresolvedDrsUrl(record.get(0), record.get(1))).toList + finally + csvParser.close() + } + + def runLocalizer(commandLineArguments: CommandLineArguments, drsCredentials: DrsCredentials): IO[ExitCode] = { + val urlList = + (commandLineArguments.manifestPath, commandLineArguments.drsObject, commandLineArguments.containerPath) match { + case (Some(manifestPath), _, _) => + loadCSVManifest(manifestPath) + case (_, Some(drsObject), Some(containerPath)) => + IO.pure(List(UnresolvedDrsUrl(drsObject, containerPath))) + case (_, _, _) => + throw new RuntimeException("Illegal command line arguments supplied to drs localizer.") + } + val main = new DrsLocalizerMain(urlList, + defaultDownloaderFactory, + drsCredentials, + commandLineArguments.googleRequesterPaysProject + ) + main.resolveAndDownload().map(_.exitCode) } + + /** + * Helper function to decide which downloader to use based on data from the DRS response. + * Throws a runtime exception if the DRS response is invalid. + */ + def toValidatedUriType(accessUrl: Option[AccessUrl], gsUri: Option[String]): URIType = + // if both are provided, prefer using access urls + (accessUrl, gsUri) match { + case (Some(_), _) => + if (!accessUrl.get.url.startsWith("https://")) { + throw new RuntimeException("Resolved Access URL does not start with https://") + } + URIType.ACCESS + case (_, Some(_)) => + if (!gsUri.get.startsWith("gs://")) { + throw new RuntimeException("Resolved Google URL does not start with gs://") + } + URIType.GCS + case (_, _) => + throw new RuntimeException("DRS response did not contain any URLs") + } +} + +object URIType extends Enumeration { + type URIType = Value + val GCS, ACCESS, UNKNOWN = Value } -class DrsLocalizerMain(drsUrl: String, - downloadLoc: String, - accessTokenStrategy: AccessTokenStrategy, - requesterPaysProjectIdOption: Option[String]) extends StrictLogging { +class DrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]], + downloaderFactory: DownloaderFactory, + drsCredentials: DrsCredentials, + requesterPaysProjectIdOption: Option[String] +) extends StrictLogging { + + /** + * This will: + * - resolve all URLS + * - build downloader(s) for them + * - Invoke the downloaders to localize the files. + * @return DownloadSuccess if all downloads succeed. An error otherwise. + */ + def resolveAndDownload(): IO[DownloadResult] = { + val downloadResults = buildDownloaders().flatMap { downloaderList => + downloaderList.map(downloader => downloader.download).traverse(identity) + } + downloadResults.map { list => + list.find(result => result != DownloadSuccess).getOrElse(DownloadSuccess) + } + } - def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = { + def getDrsPathResolver: IO[DrsPathResolver] = IO { val drsConfig = DrsConfig.fromEnv(sys.env) - new DrsLocalizerDrsPathResolver(drsConfig, accessTokenStrategy) + logger.info(s"Using ${drsConfig.drsResolverUrl} to resolve DRS Objects") + new DrsPathResolver(drsConfig, drsCredentials) } - } - def resolveAndDownloadWithRetries(downloadRetries: Int, - checksumRetries: Int, - downloaderFactory: DownloaderFactory, - backoff: Option[CloudNioBackoff], - downloadAttempt: Int = 0, - checksumAttempt: Int = 0): IO[DownloadResult] = { + /** + * After resolving all of the URLs, this sorts them into an "Access" or "GCS" bucket. + * All access URLS will be downloaded as a batch with a single bulk downloader. + * All google URLs will be downloaded individually in their own google downloader. + * @return List of all downloaders required to fulfill the request. + */ + def buildDownloaders(): IO[List[Downloader]] = + resolveUrls(toResolveAndDownload).map { pendingDownloads => + val accessUrls = pendingDownloads.filter(url => url.uriType == URIType.ACCESS) + val googleUrls = pendingDownloads.filter(url => url.uriType == URIType.GCS) + val bulkDownloader: List[Downloader] = + if (accessUrls.isEmpty) List() else List(buildBulkAccessUrlDownloader(accessUrls)) + val googleDownloaders: List[Downloader] = if (googleUrls.isEmpty) List() else buildGoogleDownloaders(googleUrls) + bulkDownloader ++ googleDownloaders + } - def maybeRetryForChecksumFailure(t: Throwable): IO[DownloadResult] = { - if (checksumAttempt < checksumRetries) { - backoff foreach { b => Thread.sleep(b.backoffMillis) } - logger.warn(s"Attempting retry $checksumAttempt of $checksumRetries checksum retries to download $drsUrl", t) - // In the event of a checksum failure reset the download attempt to zero. - resolveAndDownloadWithRetries(downloadRetries, checksumRetries, downloaderFactory, backoff map { _.next }, 0, checksumAttempt + 1) - } else { - IO.raiseError(new RuntimeException(s"Exhausted $checksumRetries checksum retries to resolve, download and checksum $drsUrl", t)) - } + def buildGoogleDownloaders(resolvedGoogleUrls: List[ResolvedDrsUrl]): List[Downloader] = + resolvedGoogleUrls.map { url => + downloaderFactory.buildGcsUriDownloader( + gcsPath = url.drsResponse.gsUri.get, + serviceAccountJsonOption = url.drsResponse.googleServiceAccount.map(_.data.spaces2), + downloadLoc = url.downloadDestinationPath, + requesterPaysProjectOption = requesterPaysProjectIdOption + ) } + def buildBulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]): Downloader = + downloaderFactory.buildBulkAccessUrlDownloader(resolvedUrls) + + /** + * Runs a synchronous HTTP request to resolve the provided DRS URL with the provided resolver. + */ + def resolveSingleUrl(resolverObject: DrsPathResolver, drsUrlToResolve: UnresolvedDrsUrl): IO[ResolvedDrsUrl] = { + val fields = NonEmptyList.of(DrsResolverField.GsUri, + DrsResolverField.GoogleServiceAccount, + DrsResolverField.AccessUrl, + DrsResolverField.Hashes + ) + val drsResponse = resolverObject.resolveDrs(drsUrlToResolve.drsUrl, fields) + drsResponse.map(resp => + ResolvedDrsUrl(resp, drsUrlToResolve.downloadDestinationPath, toValidatedUriType(resp.accessUrl, resp.gsUri)) + ) + } - def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = { - t match { - case _: FatalRetryDisposition => - IO.raiseError(t) - case _ if downloadAttempt < downloadRetries => - backoff foreach { b => Thread.sleep(b.backoffMillis) } - logger.warn(s"Attempting retry $downloadAttempt of $downloadRetries download retries to download $drsUrl", t) - resolveAndDownloadWithRetries(downloadRetries, checksumRetries, downloaderFactory, backoff map { _.next }, downloadAttempt + 1, checksumAttempt) - case _ => - IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries download retries to resolve, download and checksum $drsUrl", t)) + val defaultBackoff: CloudNioBackoff = + CloudNioSimpleExponentialBackoff(initialInterval = 10 seconds, maxInterval = 60 seconds, multiplier = 2) + + /** + * Runs synchronous HTTP requests to resolve all the DRS urls. + */ + def resolveUrls(unresolvedUrls: IO[List[UnresolvedDrsUrl]]): IO[List[ResolvedDrsUrl]] = + unresolvedUrls.flatMap { unresolvedList => + getDrsPathResolver.flatMap { resolver => + unresolvedList + .map { unresolvedUrl => + resolveWithRetries(resolver, unresolvedUrl, defaultNumRetries, Option(defaultBackoff)) + } + .traverse(identity) } } - resolveAndDownload(downloaderFactory).redeemWith({ - maybeRetryForDownloadFailure - }, - { - case f: FatalDownloadFailure => - IO.raiseError(new RuntimeException(s"Fatal error downloading DRS object: $f")) - case r: RetryableDownloadFailure => - maybeRetryForDownloadFailure( - new RuntimeException(s"Retryable download error: $r for $drsUrl on retry attempt $downloadAttempt of $downloadRetries") with RegularRetryDisposition) - case ChecksumFailure => - maybeRetryForChecksumFailure(new RuntimeException(s"Checksum failure for $drsUrl on checksum retry attempt $checksumAttempt of $checksumRetries")) - case o => IO.pure(o) - }) - } + def resolveWithRetries(resolverObject: DrsPathResolver, + drsUrlToResolve: UnresolvedDrsUrl, + resolutionRetries: Int, + backoff: Option[CloudNioBackoff], + resolutionAttempt: Int = 0 + ): IO[ResolvedDrsUrl] = { - private [localizer] def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = { - resolve(downloaderFactory) flatMap { _.download } - } + def maybeRetryForResolutionFailure(t: Throwable): IO[ResolvedDrsUrl] = + if (resolutionAttempt < resolutionRetries) { + backoff foreach { b => Thread.sleep(b.backoffMillis) } + logger.warn( + s"Attempting retry $resolutionAttempt of $resolutionRetries drs resolution retries to resolve ${drsUrlToResolve.drsUrl}", + t + ) + resolveWithRetries(resolverObject, + drsUrlToResolve, + resolutionRetries, + backoff map { _.next }, + resolutionAttempt + 1 + ) + } else { + IO.raiseError( + new RuntimeException(s"Exhausted $resolutionRetries resolution retries to resolve $drsUrlToResolve.drsUrl", t) + ) + } - private [localizer] def resolve(downloaderFactory: DownloaderFactory): IO[Downloader] = { - val fields = NonEmptyList.of(MarthaField.GsUri, MarthaField.GoogleServiceAccount, MarthaField.AccessUrl, MarthaField.Hashes) - for { - resolver <- getDrsPathResolver - marthaResponse <- resolver.resolveDrsThroughMartha(drsUrl, fields) - - // Currently Martha only supports resolving DRS paths to access URLs or GCS paths. - downloader <- (marthaResponse.accessUrl, marthaResponse.gsUri) match { - case (Some(accessUrl), _) => - downloaderFactory.buildAccessUrlDownloader(accessUrl, downloadLoc, marthaResponse.hashes) - case (_, Some(gcsPath)) => - val serviceAccountJsonOption = marthaResponse.googleServiceAccount.map(_.data.spaces2) - downloaderFactory.buildGcsUriDownloader( - gcsPath = gcsPath, - serviceAccountJsonOption = serviceAccountJsonOption, - downloadLoc = downloadLoc, - requesterPaysProjectOption = requesterPaysProjectIdOption) - case _ => - IO.raiseError(new RuntimeException(DrsPathResolver.ExtractUriErrorMsg)) + resolveSingleUrl(resolverObject, drsUrlToResolve).redeemWith( + recover = maybeRetryForResolutionFailure, + bind = { + case f: FatalRetryDisposition => + IO.raiseError(new RuntimeException(s"Fatal error resolving DRS URL: $f")) + case _: RegularRetryDisposition => + resolveWithRetries(resolverObject, drsUrlToResolve, resolutionRetries, backoff, resolutionAttempt + 1) + case o => IO.pure(o) } - } yield downloader + ) } } diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AccessTokenStrategy.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AccessTokenStrategy.scala deleted file mode 100644 index 756f5371c0f..00000000000 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AccessTokenStrategy.scala +++ /dev/null @@ -1,7 +0,0 @@ -package drs.localizer.accesstokens - -import common.validation.ErrorOr.ErrorOr - -trait AccessTokenStrategy { - def getAccessToken(): ErrorOr[String] -} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AzureB2CAccessTokenStrategy.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AzureB2CAccessTokenStrategy.scala deleted file mode 100644 index cf83171adf0..00000000000 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/AzureB2CAccessTokenStrategy.scala +++ /dev/null @@ -1,40 +0,0 @@ -package drs.localizer.accesstokens - -import cats.syntax.validated._ -import com.azure.identity.DefaultAzureCredentialBuilder -import com.azure.security.keyvault.secrets.{SecretClient, SecretClientBuilder} -import common.validation.ErrorOr -import common.validation.ErrorOr.{ErrorOr,ShortCircuitingFlatMap} -import drs.localizer.CommandLineArguments - -case class AzureB2CAccessTokenStrategy(commandLineArguments: CommandLineArguments) extends AccessTokenStrategy { - override def getAccessToken(): ErrorOr[String] = { - commandLineArguments match { - case CommandLineArguments(_, _, _, _, Some(vault), Some(secret), clientId) => - AzureKeyVaultClient(vault, clientId) flatMap { _.getSecret(secret) } - case invalid => s"Invalid command line arguments: $invalid".invalidNel - } - } -} - -// Note: The AzureKeyVaultClient code here is basically a copy/paste of the code temporarily living in the TES backend. -// All the current KeyVault interaction in Cromwell is temporary, but the code in the TES backend might be even more -// temporary than this. -class AzureKeyVaultClient(client: SecretClient) { - def getSecret(secretName: String): ErrorOr[String] = ErrorOr(client.getSecret(secretName).getValue) -} - -object AzureKeyVaultClient { - def apply(vaultName: String, identityClientId: Option[String]): ErrorOr[AzureKeyVaultClient] = ErrorOr { - val credentialBuilder = identityClientId.foldLeft(new DefaultAzureCredentialBuilder()) { - (builder, clientId) => builder.managedIdentityClientId(clientId) - } - - val client = new SecretClientBuilder() - .vaultUrl(s"https://${vaultName}.vault.azure.net") - .credential(credentialBuilder.build()) - .buildClient() - - new AzureKeyVaultClient(client) - } -} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/GoogleAccessTokenStrategy.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/GoogleAccessTokenStrategy.scala deleted file mode 100644 index 5af2b8af441..00000000000 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/accesstokens/GoogleAccessTokenStrategy.scala +++ /dev/null @@ -1,28 +0,0 @@ -package drs.localizer.accesstokens - -import cats.syntax.validated._ -import com.google.auth.oauth2.GoogleCredentials -import common.validation.ErrorOr.ErrorOr - -import scala.jdk.CollectionConverters._ -import scala.util.{Failure, Success, Try} - -/** - * Strategy for obtaining an access token from Google Application Default credentials that are assumed to already exist. - */ -case object GoogleAccessTokenStrategy extends AccessTokenStrategy { - private final val UserInfoEmailScope = "https://www.googleapis.com/auth/userinfo.email" - private final val UserInfoProfileScope = "https://www.googleapis.com/auth/userinfo.profile" - private final val UserInfoScopes = List(UserInfoEmailScope, UserInfoProfileScope) - - override def getAccessToken(): ErrorOr[String] = { - Try { - val scopedCredentials = GoogleCredentials.getApplicationDefault().createScoped(UserInfoScopes.asJava) - scopedCredentials.refreshAccessToken().getTokenValue - } match { - case Success(null) => "null token value attempting to refresh access token".invalidNel - case Success(value) => value.validNel - case Failure(e) => s"Failed to refresh access token: ${e.getMessage}".invalidNel - } - } -} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala deleted file mode 100644 index ae6f2fa4f1e..00000000000 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/AccessUrlDownloader.scala +++ /dev/null @@ -1,91 +0,0 @@ -package drs.localizer.downloaders - -import cats.data.Validated.{Invalid, Valid} -import cats.effect.{ExitCode, IO} -import cloud.nio.impl.drs.AccessUrl -import com.typesafe.scalalogging.StrictLogging -import common.exception.AggregatedMessageException -import common.util.StringUtil._ -import common.validation.ErrorOr.ErrorOr -import drs.localizer.downloaders.AccessUrlDownloader._ - -import scala.sys.process.{Process, ProcessLogger} -import scala.util.matching.Regex - -case class GetmResult(returnCode: Int, stderr: String) - -case class AccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes) extends Downloader with StrictLogging { - def generateDownloadScript: ErrorOr[String] = { - val signedUrl = accessUrl.url - GetmChecksum(hashes, accessUrl).args map { checksumArgs => - s"""mkdir -p $$(dirname '$downloadLoc') && rm -f '$downloadLoc' && getm $checksumArgs --filepath '$downloadLoc' '$signedUrl'""" - } - } - - def runGetm: IO[GetmResult] = { - generateDownloadScript match { - case Invalid(errors) => - IO.raiseError(AggregatedMessageException("Error generating access URL download script", errors.toList)) - case Valid(script) => IO { - val copyCommand = Seq("bash", "-c", script) - val copyProcess = Process(copyCommand) - - val stderr = new StringBuilder() - val errorCapture: String => Unit = { s => stderr.append(s); () } - - // As of `getm` version 0.0.4 the contents of stdout do not appear to be interesting (only a progress bar - // with no option to suppress it), so ignore stdout for now. If stdout becomes interesting in future versions - // of `getm` it can be captured just like stderr is being captured here. - val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture) - - GetmResult(returnCode, stderr.toString().trim()) - } - } - } - - override def download: IO[DownloadResult] = { - // We don't want to log the unmasked signed URL here. On a PAPI backend this log will end up under the user's - // workspace bucket, but that bucket may have visibility different than the data referenced by the signed URL. - val masked = accessUrl.url.maskSensitiveUri - logger.info(s"Attempting to download data to '$downloadLoc' from access URL '$masked'.") - - runGetm map toDownloadResult - } - - def toDownloadResult(getmResult: GetmResult): DownloadResult = { - getmResult match { - case GetmResult(0, stderr) if stderr.isEmpty => - DownloadSuccess - case GetmResult(0, stderr) => - stderr match { - case ChecksumFailureMessage() => - ChecksumFailure - case _ => - UnrecognizedRetryableDownloadFailure(ExitCode(0)) - } - case GetmResult(rc, stderr) => - stderr match { - case HttpStatusMessage(status) => - Integer.parseInt(status) match { - case 408 | 429 => - RecognizedRetryableDownloadFailure(ExitCode(rc)) - case s if s / 100 == 4 => - FatalDownloadFailure(ExitCode(rc)) - case s if s / 100 == 5 => - RecognizedRetryableDownloadFailure(ExitCode(rc)) - case _ => - UnrecognizedRetryableDownloadFailure(ExitCode(rc)) - } - case _ => - UnrecognizedRetryableDownloadFailure(ExitCode(rc)) - } - } - } -} - -object AccessUrlDownloader { - type Hashes = Option[Map[String, String]] - - val ChecksumFailureMessage: Regex = raw""".*AssertionError: Checksum failed!.*""".r - val HttpStatusMessage: Regex = raw"""ERROR:getm\.cli.*"status_code":\s*(\d+).*""".r -} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala new file mode 100644 index 00000000000..1fd5bc6bf60 --- /dev/null +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/BulkAccessUrlDownloader.scala @@ -0,0 +1,136 @@ +package drs.localizer.downloaders + +import cats.effect.{ExitCode, IO} +import cloud.nio.impl.drs.AccessUrl +import com.typesafe.scalalogging.StrictLogging + +import java.nio.charset.StandardCharsets +import java.nio.file.{Files, Path, Paths} +import scala.sys.process.{Process, ProcessLogger} +import scala.util.matching.Regex +import drs.localizer.ResolvedDrsUrl +import spray.json.DefaultJsonProtocol.{listFormat, mapFormat, StringJsonFormat} +import spray.json._ + +case class GetmResult(returnCode: Int, stderr: String) + +/** + * Getm is a python tool that is used to download resolved DRS uris quickly and in parallel. + * This class builds a getm-manifest.json file that it uses for input, and builds/executes a shell command + * to invoke the Getm tool, which is expected to already be installed in the local environment. + * @param resolvedUrls + */ +case class BulkAccessUrlDownloader(resolvedUrls: List[ResolvedDrsUrl]) extends Downloader with StrictLogging { + + val getmManifestPath: Path = Paths.get("getm-manifest.json") + + /** + * Write a json manifest to disk that looks like: + * // [ + * // { + * // "url" : "www.123.com", + * // "filepath" : "path/to/where/123/should/be/downloaded", + * // "checksum" : "sdfjndsfjkfsdjsdfkjsdf", + * // "checksum-algorithm" : "md5" + * // }, + * // { + * // "url" : "www.567.com" + * // "filepath" : "path/to/where/567/should/be/downloaded", + * // "checksum" : "asdasdasfsdfsdfasdsdfasd", + * // "checksum-algorithm" : "md5" + * // } + * // ] + * + * @param resolvedUrls + * @return Filepath of a getm-manifest.json that Getm can use to download multiple files in parallel. + */ + def generateJsonManifest(resolvedUrls: List[ResolvedDrsUrl]): IO[Path] = { + def resolvedUrlToJsonMap(resolvedUrl: ResolvedDrsUrl): Map[String, String] = { + val accessUrl: AccessUrl = resolvedUrl.drsResponse.accessUrl.getOrElse(AccessUrl("missing", None)) + resolvedUrl.drsResponse.hashes + .map { _ => + val checksum = + GetmChecksum(resolvedUrl.drsResponse.hashes, accessUrl).value.getOrElse("error_calculating_checksum") + val checksumAlgorithm = GetmChecksum(resolvedUrl.drsResponse.hashes, accessUrl).getmAlgorithm + Map( + ("url", accessUrl.url), + ("filepath", resolvedUrl.downloadDestinationPath), + ("checksum", checksum), + ("checksum-algorithm", checksumAlgorithm) + ) + } + .getOrElse( + Map( + ("url", accessUrl.url), + ("filepath", resolvedUrl.downloadDestinationPath) + ) + ) + } + + val jsonArray: String = resolvedUrls.map(resolved => resolvedUrlToJsonMap(resolved)).toJson.prettyPrint + IO(Files.write(getmManifestPath, jsonArray.getBytes(StandardCharsets.UTF_8))) + } + + def deleteJsonManifest() = + Files.deleteIfExists(getmManifestPath) + + def generateGetmCommand(pathToMainfestJson: Path): String = + s"""timeout 24h getm --manifest ${pathToMainfestJson.toString} -vv""" + def runGetm: IO[GetmResult] = + generateJsonManifest(resolvedUrls).flatMap { manifestPath => + val script = generateGetmCommand(manifestPath) + val copyCommand: Seq[String] = Seq("bash", "-c", script) + logger.info(script) + val copyProcess = Process(copyCommand) + val stderr = new StringBuilder() + val errorCapture: String => Unit = { s => stderr.append(s); () } + val returnCode = copyProcess ! ProcessLogger(_ => (), errorCapture) + deleteJsonManifest() + logger.info(stderr.toString().trim()) + IO(GetmResult(returnCode, stderr.toString().trim())) + } + + override def download: IO[DownloadResult] = { + // We don't want to log the unmasked signed URL here. On a PAPI backend this log will end up under the user's + // workspace bucket, but that bucket may have visibility different than the data referenced by the signed URL. + logger.info(s"Attempting to download data") + + runGetm map toDownloadResult + } + + def toDownloadResult(getmResult: GetmResult): DownloadResult = + getmResult match { + case GetmResult(0, stderr) if stderr.isEmpty => + DownloadSuccess + case GetmResult(0, stderr) => + stderr match { + case BulkAccessUrlDownloader.ChecksumFailureMessage() => + ChecksumFailure + case _ => + UnrecognizedRetryableDownloadFailure(ExitCode(0)) + } + case GetmResult(rc, stderr) => + stderr match { + case BulkAccessUrlDownloader.HttpStatusMessage(status) => + Integer.parseInt(status) match { + case 408 | 429 => + RecognizedRetryableDownloadFailure(ExitCode(rc)) + case s if s / 100 == 4 => + FatalDownloadFailure(ExitCode(rc)) + case s if s / 100 == 5 => + RecognizedRetryableDownloadFailure(ExitCode(rc)) + case _ => + UnrecognizedRetryableDownloadFailure(ExitCode(rc)) + } + case _ => + UnrecognizedRetryableDownloadFailure(ExitCode(rc)) + } + } +} + +object BulkAccessUrlDownloader { + type Hashes = Option[Map[String, String]] + + val ChecksumFailureMessage: Regex = raw""".*AssertionError: Checksum failed!.*""".r + val HttpStatusMessage: Regex = raw"""ERROR:getm\.cli.*"status_code":\s*(\d+).*""".r +} diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala index 8465ede0dd6..c35a2b1634e 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/DownloaderFactory.scala @@ -1,14 +1,13 @@ package drs.localizer.downloaders -import cats.effect.IO -import cloud.nio.impl.drs.AccessUrl -import drs.localizer.downloaders.AccessUrlDownloader.Hashes +import drs.localizer.ResolvedDrsUrl trait DownloaderFactory { - def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] + def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, - requesterPaysProjectOption: Option[String]): IO[Downloader] + requesterPaysProjectOption: Option[String] + ): Downloader } diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala index 345a14f63e4..74f5bc64621 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GcsUriDownloader.scala @@ -1,25 +1,36 @@ package drs.localizer.downloaders import cats.effect.{ExitCode, IO} +import cloud.nio.spi.{CloudNioBackoff, CloudNioSimpleExponentialBackoff} import com.typesafe.scalalogging.StrictLogging import drs.localizer.downloaders.GcsUriDownloader.RequesterPaysErrorMsg - +import scala.language.postfixOps import java.nio.charset.StandardCharsets import java.nio.file.{Files, Path} +import scala.concurrent.duration.DurationInt import scala.sys.process.{Process, ProcessLogger} case class GcsUriDownloader(gcsUrl: String, serviceAccountJson: Option[String], downloadLoc: String, - requesterPaysProjectIdOption: Option[String]) extends Downloader with StrictLogging { + requesterPaysProjectIdOption: Option[String] +) extends Downloader + with StrictLogging { + + val defaultNumRetries: Int = 5 + val defaultBackoff: CloudNioBackoff = + CloudNioSimpleExponentialBackoff(initialInterval = 1 seconds, maxInterval = 60 seconds, multiplier = 2) + + override def download: IO[DownloadResult] = + downloadWithRetries(defaultNumRetries, Option(defaultBackoff)) - override def download: IO[DownloadResult] = { + def runDownloadCommand: IO[DownloadResult] = { logger.info(s"Requester Pays project ID is $requesterPaysProjectIdOption") logger.info(s"Attempting to download $gcsUrl to $downloadLoc") val copyProcess = serviceAccountJson match { case Some(sa) => - // if Martha returned a SA, use that SA for gsutil instead of default credentials + // if DRS Resolver returned a SA, use that SA for gsutil instead of default credentials val tempCredentialDir: Path = Files.createTempDirectory("gcloudTemp_").toAbsolutePath val saJsonPath: Path = tempCredentialDir.resolve("sa.json") Files.write(saJsonPath, sa.getBytes(StandardCharsets.UTF_8)) @@ -27,7 +38,7 @@ case class GcsUriDownloader(gcsUrl: String, val copyCommand = Seq("bash", "-c", generateDownloadScript(gcsUrl, Option(saJsonPath))) Process(copyCommand, None, extraEnv.toSeq: _*) case None => - // No SA returned from Martha. gsutil will use the application default credentials. + // No SA returned from DRS Resolver. gsutil will use the application default credentials. val copyCommand = Seq("bash", "-c", generateDownloadScript(gcsUrl, None)) Process(copyCommand) } @@ -35,11 +46,46 @@ case class GcsUriDownloader(gcsUrl: String, // run the multiple bash script to download file and log stream sent to stdout and stderr using ProcessLogger val returnCode = copyProcess ! ProcessLogger(logger.underlying.info, logger.underlying.error) - val result = if (returnCode == 0) DownloadSuccess else RecognizedRetryableDownloadFailure(exitCode = ExitCode(returnCode)) + val result = + if (returnCode == 0) DownloadSuccess else RecognizedRetryableDownloadFailure(exitCode = ExitCode(returnCode)) IO.pure(result) } + def downloadWithRetries(downloadRetries: Int, + backoff: Option[CloudNioBackoff], + downloadAttempt: Int = 0 + ): IO[DownloadResult] = { + + def maybeRetryForDownloadFailure(t: Throwable): IO[DownloadResult] = + if (downloadAttempt < downloadRetries) { + backoff foreach { b => Thread.sleep(b.backoffMillis) } + logger.warn(s"Attempting download retry $downloadAttempt of $downloadRetries for a GCS url", t) + downloadWithRetries(downloadRetries, + backoff map { + _.next + }, + downloadAttempt + 1 + ) + } else { + IO.raiseError(new RuntimeException(s"Exhausted $downloadRetries resolution retries to download GCS file", t)) + } + + runDownloadCommand.redeemWith( + recover = maybeRetryForDownloadFailure, + bind = { + case s: DownloadSuccess.type => + IO.pure(s) + case _: RecognizedRetryableDownloadFailure => + downloadWithRetries(downloadRetries, backoff, downloadAttempt + 1) + case _: UnrecognizedRetryableDownloadFailure => + downloadWithRetries(downloadRetries, backoff, downloadAttempt + 1) + case _ => + downloadWithRetries(downloadRetries, backoff, downloadAttempt + 1) + } + ) + } + /** * Bash to download the GCS file using `gsutil`. */ @@ -47,14 +93,14 @@ case class GcsUriDownloader(gcsUrl: String, def gcsCopyCommand(flag: String = ""): String = s"gsutil $flag cp $gcsUrl $downloadLoc" - def setServiceAccount(): String = { + def setServiceAccount(): String = saJsonPathOption match { case Some(saJsonPath) => - s"""# Set gsutil to use the service account returned from Martha + s"""# Set gsutil to use the service account returned from the DRS Resolver |gcloud auth activate-service-account --key-file=$saJsonPath > gcloud_output.txt 2>&1 |RC_GCLOUD=$$? |if [ "$$RC_GCLOUD" != "0" ]; then - | echo "Failed to activate service account returned from Martha. File won't be downloaded. Error: $$(cat gcloud_output.txt)" >&2 + | echo "Failed to activate service account returned from the DRS Resolver. File won't be downloaded. Error: $$(cat gcloud_output.txt)" >&2 | exit "$$RC_GCLOUD" |else | echo "Successfully activated service account; Will continue with download. $$(cat gcloud_output.txt)" @@ -62,9 +108,8 @@ case class GcsUriDownloader(gcsUrl: String, |""".stripMargin case None => "" } - } - def recoverWithRequesterPays(): String = { + def recoverWithRequesterPays(): String = requesterPaysProjectIdOption match { case Some(userProject) => s"""if [ "$$RC_GSUTIL" != "0" ]; then @@ -78,7 +123,6 @@ case class GcsUriDownloader(gcsUrl: String, |""".stripMargin case None => "" } - } // bash to download the GCS file using gsutil s"""set -euo pipefail @@ -104,5 +148,5 @@ case class GcsUriDownloader(gcsUrl: String, } object GcsUriDownloader { - private final val RequesterPaysErrorMsg = "requester pays bucket but no user project" + final private val RequesterPaysErrorMsg = "requester pays bucket but no user project" } diff --git a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala index 252971c0b5e..a72459cdb7b 100644 --- a/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala +++ b/cromwell-drs-localizer/src/main/scala/drs/localizer/downloaders/GetmChecksum.scala @@ -3,18 +3,16 @@ package drs.localizer.downloaders import cats.syntax.validated._ import cloud.nio.impl.drs.AccessUrl import common.validation.ErrorOr.ErrorOr -import drs.localizer.downloaders.AccessUrlDownloader.Hashes -import mouse.all.anySyntaxMouse -import org.apache.commons.codec.binary.Base64.{decodeBase64, isBase64} -import org.apache.commons.codec.binary.Hex.encodeHexString +import drs.localizer.downloaders.BulkAccessUrlDownloader.Hashes +import org.apache.commons.codec.binary.Base64.encodeBase64String +import org.apache.commons.codec.binary.Hex.decodeHex import org.apache.commons.text.StringEscapeUtils - sealed trait GetmChecksum { def getmAlgorithm: String def rawValue: String def value: ErrorOr[String] = rawValue.validNel - def args: ErrorOr[String] = { + def args: ErrorOr[String] = // The value for `--checksum-algorithm` is constrained by the algorithm names in the `sealed` hierarchy of // `GetmChecksum`, but the value for `--checksum` is largely a function of data returned by the DRS server. // Shell escape this to avoid injection. @@ -22,27 +20,22 @@ sealed trait GetmChecksum { val escapedValue = StringEscapeUtils.escapeXSI(v) s"--checksum-algorithm '$getmAlgorithm' --checksum $escapedValue" } - } } case class Md5(override val rawValue: String) extends GetmChecksum { - override def value: ErrorOr[String] = { - val trimmed = rawValue.trim - if (trimmed.matches("[A-Fa-f0-9]+")) - trimmed.validNel - // TDR currently returns a base64-encoded MD5 because that's what Azure seems to do. However, - // the DRS spec does not specify that any checksums should be base64-encoded, and `getm` also - // does not expect base64. This case handles the current behavior in the short term until - // https://broadworkbench.atlassian.net/browse/DR-2259 is done. - else if (isBase64(trimmed)) - (trimmed |> decodeBase64 |> encodeHexString).validNel - else - s"Invalid md5 checksum value is neither hex nor base64: $rawValue".invalidNel - } + override def value: ErrorOr[String] = GetmChecksum.validateHex(rawValue) override def getmAlgorithm: String = "md5" } case class Crc32c(override val rawValue: String) extends GetmChecksum { + // The DRS spec says that all hash values should be hex strings, + // but getm expects crc32c values to be base64. + override def value: ErrorOr[String] = + GetmChecksum + .validateHex(rawValue) + .map(decodeHex) + .map(encodeBase64String) + override def getmAlgorithm: String = "gs_crc32c" } case class AwsEtag(override val rawValue: String) extends GetmChecksum { @@ -58,17 +51,16 @@ case class Unsupported(override val rawValue: String) extends GetmChecksum { } object GetmChecksum { - def apply(hashes: Hashes, accessUrl: AccessUrl): GetmChecksum = { + def apply(hashes: Hashes, accessUrl: AccessUrl): GetmChecksum = hashes match { case Some(hashes) if hashes.nonEmpty => - // `hashes` is keyed by the Martha names for these hash algorithms, which in turn are the forwarded DRS + // `hashes` is keyed by the DRS Resolver names for these hash algorithms, which in turn are the forwarded DRS // providers' names for the algorithms. `getm` has its own notions of what these algorithms are called. // For the specific case of `md5` the algorithm names are the same between DRS providers and `getm`, // but all of the other algorithm names currently differ between DRS providers and `getm`. if (hashes.contains("md5")) { Md5(hashes("md5")) - } - else if (hashes.contains("crc32c")) { + } else if (hashes.contains("crc32c")) { Crc32c(hashes("crc32c")) } // etags could be anything; only ask `getm` to check s3 etags if this actually looks like an s3 signed url. @@ -87,5 +79,12 @@ object GetmChecksum { } case _ => Null // None or an empty hashes map. } + + def validateHex(s: String): ErrorOr[String] = { + val trimmed = s.trim + if (trimmed.matches("[A-Fa-f0-9]+")) + trimmed.validNel + else + s"Invalid checksum value, expected hex but got: $trimmed".invalidNel } } diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala index 3f7b96f5edc..6658428e650 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/CommandLineParserSpec.scala @@ -13,9 +13,8 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma private val drsObject = "drs://provider/object" private val containerPath = "/cromwell_root/my.bam" private val requesterPaysProject = "i_heart_egress" - private val azureVaultName = "Kwikset" - private val azureSecretName = "shhh" private val azureIdentityClientId = "itme@azure.com" + private val manifestPath = "/my/manifest.txt" behavior of "DRS Localizer command line parser" @@ -38,106 +37,148 @@ class CommandLineParserSpec extends AnyFlatSpec with CromwellTimeoutSpec with Ma args.containerPath.get shouldBe containerPath args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google args.googleRequesterPaysProject shouldBe empty - args.azureVaultName shouldBe empty - args.azureSecretName shouldBe empty args.azureIdentityClientId shouldBe empty + args.manifestPath shouldBe empty } it should "successfully parse with three arguments" in { val args = parser.parse(Array(drsObject, containerPath, requesterPaysProject), CommandLineArguments()).get + args.drsObject.get shouldBe drsObject + args.containerPath.get shouldBe containerPath + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google + args.googleRequesterPaysProject.get shouldBe requesterPaysProject + args.azureIdentityClientId shouldBe empty + args.manifestPath shouldBe empty + } + + it should "successfully parse requester pays project" in { + val args = parser.parse(Array(drsObject, containerPath, "-r", requesterPaysProject), CommandLineArguments()).get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google args.googleRequesterPaysProject.get shouldBe requesterPaysProject - args.azureVaultName shouldBe empty - args.azureSecretName shouldBe empty args.azureIdentityClientId shouldBe empty + args.manifestPath shouldBe empty } - it should "successfully parse an explicit Google access token strategy invocation" in { - val args = parser.parse(Array("--access-token-strategy", "google", drsObject, containerPath, requesterPaysProject), CommandLineArguments()).get + it should "successfully parse with three arguments and requester pays project" in { + val args = parser + .parse(Array(drsObject, containerPath, requesterPaysProject, "-r", requesterPaysProject), CommandLineArguments()) + .get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google args.googleRequesterPaysProject.get shouldBe requesterPaysProject - args.azureVaultName shouldBe empty - args.azureSecretName shouldBe empty args.azureIdentityClientId shouldBe empty + args.manifestPath shouldBe empty + } + + it should "fail if requester pays argument and flag specify different projects" in { + parser.parse(Array(drsObject, containerPath, requesterPaysProject, "-r", "boom!"), + CommandLineArguments() + ) shouldBe None } - it should "fail to parse an Azure invocation missing vault name and secret name" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - drsObject, containerPath), CommandLineArguments()) + it should "successfully parse args with a manifest file" in { + val args = parser.parse(Array("-m", manifestPath), CommandLineArguments()).get - args shouldBe None + args.drsObject shouldBe empty + args.containerPath shouldBe empty + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google + args.googleRequesterPaysProject shouldBe empty + args.azureIdentityClientId shouldBe empty + args.manifestPath.get shouldBe manifestPath } - it should "fail to parse an Azure invocation missing vault name" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - "--secret-name", azureSecretName, - drsObject, containerPath), CommandLineArguments()) + it should "fail to parse with a manifest file and one single-file arg" in { + val args = parser.parse(Array(drsObject, "--manifest-path", manifestPath), CommandLineArguments()) + args shouldBe None + } + it should "fail to parse with a manifest file and two single-file args" in { + val args = parser.parse(Array(drsObject, containerPath, "--manifest-path", manifestPath), CommandLineArguments()) args shouldBe None } - it should "fail to parse an Azure invocation missing secret name" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - "--vault-name", azureVaultName, - drsObject, containerPath), CommandLineArguments()) + it should "successfully parse an explicit Google access token strategy invocation" in { + val args = parser + .parse(Array( + "--access-token-strategy", + "google", + drsObject, + containerPath, + "--requester-pays-project", + requesterPaysProject + ), + CommandLineArguments() + ) + .get - args shouldBe None + args.drsObject.get shouldBe drsObject + args.containerPath.get shouldBe containerPath + args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Google + args.googleRequesterPaysProject.get shouldBe requesterPaysProject + args.azureIdentityClientId shouldBe empty + args.manifestPath shouldBe empty } it should "fail to parse an Azure invocation that specifies requester pays" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - "--secret-name", azureSecretName, - "--vault-name", azureVaultName, - drsObject, containerPath, requesterPaysProject), CommandLineArguments()) + val args = parser.parse( + Array("--access-token-strategy", + AccessTokenStrategy.Azure, + drsObject, + containerPath, + "--requester-pays-project", + requesterPaysProject + ), + CommandLineArguments() + ) args shouldBe None } it should "successfully parse an Azure invocation" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - "--secret-name", azureSecretName, - "--vault-name", azureVaultName, - drsObject, containerPath), CommandLineArguments()).get + val args = parser + .parse(Array("--access-token-strategy", AccessTokenStrategy.Azure, drsObject, containerPath), + CommandLineArguments() + ) + .get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Azure args.googleRequesterPaysProject shouldBe empty - args.azureVaultName.get shouldBe azureVaultName - args.azureSecretName.get shouldBe azureSecretName args.azureIdentityClientId shouldBe empty + args.manifestPath shouldBe empty } - it should "successfully parse an Azure invocation with all the trimmings" in { - val args = parser.parse(Array( - "--access-token-strategy", AccessTokenStrategy.Azure, - "--vault-name", azureVaultName, - "--secret-name", azureSecretName, - "--identity-client-id", azureIdentityClientId, - drsObject, containerPath), CommandLineArguments()).get + it should "successfully parse an Azure invocation with identity" in { + val args = parser + .parse( + Array("--access-token-strategy", + AccessTokenStrategy.Azure, + "--identity-client-id", + azureIdentityClientId, + drsObject, + containerPath + ), + CommandLineArguments() + ) + .get args.drsObject.get shouldBe drsObject args.containerPath.get shouldBe containerPath args.accessTokenStrategy.get shouldBe AccessTokenStrategy.Azure args.googleRequesterPaysProject shouldBe empty - args.azureVaultName.get shouldBe azureVaultName - args.azureSecretName.get shouldBe azureSecretName args.azureIdentityClientId.get shouldBe azureIdentityClientId + args.manifestPath shouldBe empty } it should "fail to parse with an unrecognized access token strategy" in { - val args = parser.parse(Array("--access-token-strategy", "nebulous", drsObject, containerPath), CommandLineArguments()) + val args = + parser.parse(Array("--access-token-strategy", "nebulous", drsObject, containerPath), CommandLineArguments()) args shouldBe None } } diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala index 84b407cec41..dc06ad21a0c 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/DrsLocalizerMainSpec.scala @@ -3,22 +3,50 @@ package drs.localizer import cats.data.NonEmptyList import cats.effect.{ExitCode, IO} import cats.syntax.validated._ -import cloud.nio.impl.drs.DrsPathResolver.FatalRetryDisposition -import cloud.nio.impl.drs.{AccessUrl, DrsConfig, MarthaField, MarthaResponse} +import drs.localizer.MockDrsPaths.{fakeAccessUrls, fakeDrsUrlWithGcsResolutionOnly, fakeGoogleUrls} +import cloud.nio.impl.drs.{AccessUrl, DrsConfig, DrsCredentials, DrsPathResolver, DrsResolverField, DrsResolverResponse} import common.assertion.CromwellTimeoutSpec +import common.validation.ErrorOr.ErrorOr import drs.localizer.MockDrsLocalizerDrsPathResolver.{FakeAccessTokenStrategy, FakeHashes} -import drs.localizer.accesstokens.AccessTokenStrategy -import drs.localizer.downloaders.AccessUrlDownloader.Hashes import drs.localizer.downloaders._ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { val fakeDownloadLocation = "/root/foo/foo-123.bam" val fakeRequesterPaysId = "fake-billing-project" + val fakeGoogleInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl(fakeDrsUrlWithGcsResolutionOnly, "/path/to/nowhere") + ) + ) + + val fakeAccessInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl("https://my-fake-access-url.com", "/path/to/somewhereelse") + ) + ) + + val fakeBulkGoogleInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl("drs://my-fake-google-url.com", "/path/to/nowhere"), + UnresolvedDrsUrl("drs://my-fake-google-url.com2", "/path/to/nowhere2"), + UnresolvedDrsUrl("drs://my-fake-google-url.com3", "/path/to/nowhere3"), + UnresolvedDrsUrl("drs://my-fake-google-url.com4", "/path/to/nowhere4") + ) + ) + + val fakeBulkAccessInput: IO[List[UnresolvedDrsUrl]] = IO( + List( + UnresolvedDrsUrl("drs://my-fake-access-url.com", "/path/to/somewhereelse"), + UnresolvedDrsUrl("drs://my-fake-access-url2.com", "/path/to/somewhereelse2"), + UnresolvedDrsUrl("drs://my-fake-access-url3.com", "/path/to/somewhereelse3"), + UnresolvedDrsUrl("drs://my-fake-access-url4.com", "/path/to/somewhereelse4") + ) + ) + behavior of "DrsLocalizerMain" it should "fail if drs input is not passed" in { @@ -29,264 +57,250 @@ class DrsLocalizerMainSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat DrsLocalizerMain.run(List(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly)).unsafeRunSync() shouldBe ExitCode.Error } - it should "accept arguments and run successfully without Requester Pays ID" in { - val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) - val expected = GcsUriDownloader( - gcsUrl = "gs://abc/foo-123/abc123", - serviceAccountJson = None, - downloadLoc = fakeDownloadLocation, - requesterPaysProjectIdOption = None) - mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected - } - - it should "run successfully with all 3 arguments" in { - val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, Option(fakeRequesterPaysId)) - val expected = GcsUriDownloader( - gcsUrl = "gs://abc/foo-123/abc123", - serviceAccountJson = None, - downloadLoc = fakeDownloadLocation, - requesterPaysProjectIdOption = Option(fakeRequesterPaysId)) - mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected - } - - it should "fail and throw error if Martha response does not have gs:// url" in { - val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithoutAnyResolution, fakeDownloadLocation, None) - - the[RuntimeException] thrownBy { - mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() - } should have message "No access URL nor GCS URI starting with 'gs://' found in Martha response!" + it should "tolerate no URLs being provided" in { + val mockDownloadFactory = new DownloaderFactory { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = + // This test path should never ask for the Google downloader + throw new RuntimeException("test failure111") + + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = + // This test path should never ask for the Bulk downloader + throw new RuntimeException("test failure111") + } + val mockdrsLocalizer = + new MockDrsLocalizerMain(IO(List()), mockDownloadFactory, FakeAccessTokenStrategy, Option(fakeRequesterPaysId)) + val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() + downloaders.length shouldBe 0 } - it should "resolve to use the correct downloader for an access url" in { - val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) - val expected = AccessUrlDownloader( - accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None), - downloadLoc = fakeDownloadLocation, - hashes = FakeHashes - ) - mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected - } + it should "build correct downloader(s) for a single google URL" in { + val mockDownloadFactory = new DownloaderFactory { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = + GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption) + + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = + // This test path should never ask for the Bulk downloader + throw new RuntimeException("test failure111") + } - it should "resolve to use the correct downloader for an access url when the Martha response also contains a gs url" in { - val mockDrsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlAndGcsResolution, fakeDownloadLocation, None) - val expected = AccessUrlDownloader( - accessUrl = AccessUrl(url = "http://abc/def/ghi.bam", headers = None), downloadLoc = fakeDownloadLocation, - hashes = FakeHashes + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeGoogleUrls.head._1)), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) ) - mockDrsLocalizer.resolve(DrsLocalizerMain.defaultDownloaderFactory).unsafeRunSync() shouldBe expected - } - - it should "not retry on access URL download success" in { - var actualAttempts = 0 + val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() + downloaders.length shouldBe 1 - val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) { - override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = { - actualAttempts = actualAttempts + 1 - super.resolveAndDownload(downloaderFactory) - } + val correct = downloaders.head match { + case _: GcsUriDownloader => true + case _ => false } - val accessUrlDownloader = IO.pure(new Downloader { - override def download: IO[DownloadResult] = - IO.pure(DownloadSuccess) - }) - - val downloaderFactory = new DownloaderFactory { - override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = { - accessUrlDownloader - } + correct shouldBe true + } - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = { + it should "build correct downloader(s) for a single access URL" in { + val mockDownloadFactory = new DownloaderFactory { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = // This test path should never ask for the GCS downloader throw new RuntimeException("test failure") - } + + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = + BulkAccessUrlDownloader(urlsToDownload) } - drsLocalizer.resolveAndDownloadWithRetries( - downloadRetries = 3, - checksumRetries = 1, - downloaderFactory = downloaderFactory, - backoff = None - ).unsafeRunSync() shouldBe DownloadSuccess + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(List(fakeAccessUrls.head._1)), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) + val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() + downloaders.length shouldBe 1 - actualAttempts shouldBe 1 + val expected = BulkAccessUrlDownloader( + List(fakeAccessUrls.head._2) + ) + expected shouldEqual downloaders.head } - it should "retry an appropriate number of times for regular retryable access URL download failures" in { - var actualAttempts = 0 + it should "build correct downloader(s) for multiple google URLs" in { + val mockDownloadFactory = new DownloaderFactory { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = + GcsUriDownloader(gcsPath, serviceAccountJsonOption, downloadLoc, requesterPaysProjectOption) - val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) { - override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = { - actualAttempts = actualAttempts + 1 - super.resolveAndDownload(downloaderFactory) - } - } - val accessUrlDownloader = IO.pure(new Downloader { - override def download: IO[DownloadResult] = - IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0))) - }) - - val downloaderFactory = new DownloaderFactory { - override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = { - accessUrlDownloader - } - - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = { + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = // This test path should never ask for the GCS downloader throw new RuntimeException("test failure") - } - } - - assertThrows[Throwable] { - drsLocalizer.resolveAndDownloadWithRetries( - downloadRetries = 3, - checksumRetries = 1, - downloaderFactory = downloaderFactory, - backoff = None - ).unsafeRunSync() } + val unresolvedUrls: List[UnresolvedDrsUrl] = fakeGoogleUrls.map(pair => pair._1).toList + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) + val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() + downloaders.length shouldBe unresolvedUrls.length - actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts - } - - it should "retry an appropriate number of times for fatal retryable access URL download failures" in { - var actualAttempts = 0 - - val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) { - override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = { - actualAttempts = actualAttempts + 1 - IO.raiseError(new RuntimeException("testing: fatal error") with FatalRetryDisposition) - } - } - - val accessUrlDownloader = IO.pure(new Downloader { - override def download: IO[DownloadResult] = - IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(0))) - }) - - val downloaderFactory = new DownloaderFactory { - override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = { - accessUrlDownloader + val countGoogleDownloaders = downloaders.count(downloader => + downloader match { + case _: GcsUriDownloader => true + case _ => false } + ) + // We expect one GCS downloader for each GCS uri provided + countGoogleDownloaders shouldBe downloaders.length + } - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = { + it should "build a single bulk downloader for multiple access URLs" in { + val mockDownloadFactory = new DownloaderFactory { + override def buildGcsUriDownloader(gcsPath: String, + serviceAccountJsonOption: Option[String], + downloadLoc: String, + requesterPaysProjectOption: Option[String] + ): Downloader = // This test path should never ask for the GCS downloader throw new RuntimeException("test failure") - } - } - - assertThrows[Throwable] { - drsLocalizer.resolveAndDownloadWithRetries( - downloadRetries = 3, - checksumRetries = 1, - downloaderFactory = downloaderFactory, - backoff = None - ).unsafeRunSync() - } - - actualAttempts shouldBe 1 // 1 and done with a fatal exception - } - it should "not retry on GCS URI download success" in { - var actualAttempts = 0 - val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) { - override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = { - actualAttempts = actualAttempts + 1 - super.resolveAndDownload(downloaderFactory) - } + override def buildBulkAccessUrlDownloader(urlsToDownload: List[ResolvedDrsUrl]): Downloader = + BulkAccessUrlDownloader(urlsToDownload) } - val gcsUriDownloader = IO.pure(new Downloader { - override def download: IO[DownloadResult] = - IO.pure(DownloadSuccess) - }) - - val downloaderFactory = new DownloaderFactory { - override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = { - // This test path should never ask for the access URL downloader - throw new RuntimeException("test failure") - } + val unresolvedUrls: List[UnresolvedDrsUrl] = fakeAccessUrls.map(pair => pair._1).toList + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), + mockDownloadFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) + val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() + downloaders.length shouldBe 1 - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = { - gcsUriDownloader + val countBulkDownloaders = downloaders.count(downloader => + downloader match { + case _: BulkAccessUrlDownloader => true + case _ => false } - } + ) + // We expect one total Bulk downloader for all access URIs to share + countBulkDownloaders shouldBe 1 + val expected = BulkAccessUrlDownloader( + fakeAccessUrls.map(pair => pair._2).toList + ) + expected shouldEqual downloaders.head + } - drsLocalizer.resolveAndDownloadWithRetries( - downloadRetries = 3, - checksumRetries = 1, - downloaderFactory = downloaderFactory, - backoff = None).unsafeRunSync() + it should "build 1 bulk downloader and 5 google downloaders for a mix of URLs" in { + val unresolvedUrls: List[UnresolvedDrsUrl] = + fakeAccessUrls.map(pair => pair._1).toList ++ fakeGoogleUrls.map(pair => pair._1).toList + val mockdrsLocalizer = new MockDrsLocalizerMain(IO(unresolvedUrls), + DrsLocalizerMain.defaultDownloaderFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) + val downloaders: List[Downloader] = mockdrsLocalizer.buildDownloaders().unsafeRunSync() - actualAttempts shouldBe 1 - } + downloaders.length shouldBe 6 - it should "retry an appropriate number of times for retryable GCS URI download failures" in { - var actualAttempts = 0 - val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly, fakeDownloadLocation, None) { - override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = { - actualAttempts = actualAttempts + 1 - super.resolveAndDownload(downloaderFactory) + // we expect a single bulk downloader despite 5 access URLs being provided + val countBulkDownloaders = downloaders.count(downloader => + downloader match { + case _: BulkAccessUrlDownloader => true + case _ => false } - } - val gcsUriDownloader = IO.pure(new Downloader { - override def download: IO[DownloadResult] = - IO.pure(RecognizedRetryableDownloadFailure(exitCode = ExitCode(1))) - }) - - val downloaderFactory = new DownloaderFactory { - override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = { - // This test path should never ask for the access URL downloader - throw new RuntimeException("test failure") + ) + // We expect one GCS downloader for each GCS uri provided + countBulkDownloaders shouldBe 1 + val countGoogleDownloaders = downloaders.count(downloader => + downloader match { + case _: GcsUriDownloader => true + case _ => false } + ) + // We expect one GCS downloader for each GCS uri provided + countBulkDownloaders shouldBe 1 + countGoogleDownloaders shouldBe 5 + } - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = { - gcsUriDownloader - } - } + it should "accept arguments and run successfully without Requester Pays ID" in { + val unresolved = fakeGoogleUrls.head._1 + val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), + DrsLocalizerMain.defaultDownloaderFactory, + FakeAccessTokenStrategy, + None + ) + val expected = GcsUriDownloader( + gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get, + serviceAccountJson = None, + downloadLoc = unresolved.downloadDestinationPath, + requesterPaysProjectIdOption = None + ) + val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head + downloader shouldBe expected + } - assertThrows[Throwable] { - drsLocalizer.resolveAndDownloadWithRetries( - downloadRetries = 3, - checksumRetries = 1, - downloaderFactory = downloaderFactory, - backoff = None).unsafeRunSync() - } + it should "run successfully with all 3 arguments" in { + val unresolved = fakeGoogleUrls.head._1 + val mockDrsLocalizer = new MockDrsLocalizerMain(IO(List(unresolved)), + DrsLocalizerMain.defaultDownloaderFactory, + FakeAccessTokenStrategy, + Option(fakeRequesterPaysId) + ) + val expected = GcsUriDownloader( + gcsUrl = fakeGoogleUrls.get(unresolved).get.drsResponse.gsUri.get, + serviceAccountJson = None, + downloadLoc = unresolved.downloadDestinationPath, + requesterPaysProjectIdOption = Option(fakeRequesterPaysId) + ) + val downloader: Downloader = mockDrsLocalizer.buildDownloaders().unsafeRunSync().head + downloader shouldBe expected + } - actualAttempts shouldBe 4 // 1 initial attempt + 3 retries = 4 total attempts + it should "successfully identify uri types, preferring access" in { + val exampleAccessResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("https://something.com", FakeHashes))) + val exampleGoogleResponse = DrsResolverResponse(gsUri = Option("gs://something")) + val exampleMixedResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("https://something.com", FakeHashes)), + gsUri = Option("gs://something") + ) + DrsLocalizerMain.toValidatedUriType(exampleAccessResponse.accessUrl, + exampleAccessResponse.gsUri + ) shouldBe URIType.ACCESS + DrsLocalizerMain.toValidatedUriType(exampleGoogleResponse.accessUrl, + exampleGoogleResponse.gsUri + ) shouldBe URIType.GCS + DrsLocalizerMain.toValidatedUriType(exampleMixedResponse.accessUrl, + exampleMixedResponse.gsUri + ) shouldBe URIType.ACCESS } - it should "retry an appropriate number of times for checksum failures" in { - var actualAttempts = 0 - val drsLocalizer = new MockDrsLocalizerMain(MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly, fakeDownloadLocation, None) { - override def resolveAndDownload(downloaderFactory: DownloaderFactory): IO[DownloadResult] = { - actualAttempts = actualAttempts + 1 - super.resolveAndDownload(downloaderFactory) - } - } - val accessUrlDownloader = IO.pure(new Downloader { - override def download: IO[DownloadResult] = - IO.pure(ChecksumFailure) - }) - - val downloaderFactory = new DownloaderFactory { - override def buildAccessUrlDownloader(accessUrl: AccessUrl, downloadLoc: String, hashes: Hashes): IO[Downloader] = { - accessUrlDownloader - } + it should "throw an exception if the DRS Resolver response is invalid" in { + val badAccessResponse = DrsResolverResponse(accessUrl = Option(AccessUrl("hQQps://something.com", FakeHashes))) + val badGoogleResponse = DrsResolverResponse(gsUri = Option("gQQs://something")) + val emptyResponse = DrsResolverResponse() - override def buildGcsUriDownloader(gcsPath: String, serviceAccountJsonOption: Option[String], downloadLoc: String, requesterPaysProjectOption: Option[String]): IO[Downloader] = { - // This test path should never ask for the GCS URI downloader. - throw new RuntimeException("test failure") - } - } + the[RuntimeException] thrownBy { + DrsLocalizerMain.toValidatedUriType(badAccessResponse.accessUrl, badAccessResponse.gsUri) + } should have message "Resolved Access URL does not start with https://" - assertThrows[Throwable] { - drsLocalizer.resolveAndDownloadWithRetries( - downloadRetries = 3, - checksumRetries = 1, - downloaderFactory = downloaderFactory, - backoff = None).unsafeRunSync() - } + the[RuntimeException] thrownBy { + DrsLocalizerMain.toValidatedUriType(badGoogleResponse.accessUrl, badGoogleResponse.gsUri) + } should have message "Resolved Google URL does not start with gs://" - actualAttempts shouldBe 2 // 1 initial attempt + 1 retry = 2 total attempts + the[RuntimeException] thrownBy { + DrsLocalizerMain.toValidatedUriType(emptyResponse.accessUrl, emptyResponse.gsUri) + } should have message "DRS response did not contain any URLs" } } @@ -295,45 +309,108 @@ object MockDrsPaths { val fakeDrsUrlWithAccessUrlResolutionOnly = "drs://def/bar-456/def456" val fakeDrsUrlWithAccessUrlAndGcsResolution = "drs://ghi/baz-789/ghi789" val fakeDrsUrlWithoutAnyResolution = "drs://foo/bar/no-gcs-path" -} + val fakeGoogleUrls: Map[UnresolvedDrsUrl, ResolvedDrsUrl] = Map( + (UnresolvedDrsUrl("drs://abc/foo-123/google/0", "/path/to/google/local0"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri0")), "/path/to/google/local0", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/1", "/path/to/google/local1"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri1")), "/path/to/google/local1", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/2", "/path/to/google/local2"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri2")), "/path/to/google/local2", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/3", "/path/to/google/local3"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri3")), "/path/to/google/local3", URIType.GCS) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/google/4", "/path/to/google/local4"), + ResolvedDrsUrl(DrsResolverResponse(gsUri = Option("gs://some/uri4")), "/path/to/google/local4", URIType.GCS) + ) + ) + + val fakeAccessUrls: Map[UnresolvedDrsUrl, ResolvedDrsUrl] = Map( + (UnresolvedDrsUrl("drs://abc/foo-123/access/0", "/path/to/access/local0"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/0", FakeHashes))), + "/path/to/access/local0", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/1", "/path/to/access/local1"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/1", FakeHashes))), + "/path/to/access/local1", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/2", "/path/to/access/local2"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/2", FakeHashes))), + "/path/to/access/local2", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/3", "/path/to/access/local3"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/3", FakeHashes))), + "/path/to/access/local3", + URIType.ACCESS + ) + ), + (UnresolvedDrsUrl("drs://abc/foo-123/access/4", "/path/to/access/local4"), + ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://abc/foo-123/access/4", FakeHashes))), + "/path/to/access/local4", + URIType.ACCESS + ) + ) + ) +} -class MockDrsLocalizerMain(drsUrl: String, - downloadLoc: String, - requesterPaysProjectIdOption: Option[String], - ) - extends DrsLocalizerMain(drsUrl, downloadLoc, FakeAccessTokenStrategy, requesterPaysProjectIdOption) { - - override def getDrsPathResolver: IO[DrsLocalizerDrsPathResolver] = { +class MockDrsLocalizerMain(toResolveAndDownload: IO[List[UnresolvedDrsUrl]], + downloaderFactory: DownloaderFactory, + drsCredentials: DrsCredentials, + requesterPaysProjectIdOption: Option[String] +) extends DrsLocalizerMain(toResolveAndDownload, + downloaderFactory, + FakeAccessTokenStrategy, + requesterPaysProjectIdOption + ) { + + override def getDrsPathResolver: IO[DrsPathResolver] = IO { new MockDrsLocalizerDrsPathResolver(cloud.nio.impl.drs.MockDrsPaths.mockDrsConfig) } - } + override def resolveSingleUrl(resolverObject: DrsPathResolver, + drsUrlToResolve: UnresolvedDrsUrl + ): IO[ResolvedDrsUrl] = + IO { + if (!fakeAccessUrls.contains(drsUrlToResolve) && !fakeGoogleUrls.contains(drsUrlToResolve)) { + throw new RuntimeException("Unexpected URI during testing") + } + fakeAccessUrls.getOrElse( + drsUrlToResolve, + fakeGoogleUrls.getOrElse(drsUrlToResolve, ResolvedDrsUrl(DrsResolverResponse(), "/12/3/", URIType.UNKNOWN)) + ) + } } +class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) + extends DrsPathResolver(drsConfig, FakeAccessTokenStrategy) { -class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends - DrsLocalizerDrsPathResolver(drsConfig, FakeAccessTokenStrategy) { + override def resolveDrs(drsPath: String, fields: NonEmptyList[DrsResolverField.Value]): IO[DrsResolverResponse] = { - override def resolveDrsThroughMartha(drsPath: String, fields: NonEmptyList[MarthaField.Value]): IO[MarthaResponse] = { - val marthaResponse = MarthaResponse( + val drsResolverResponse = DrsResolverResponse( size = Option(1234), hashes = FakeHashes ) IO.pure(drsPath) map { case MockDrsPaths.fakeDrsUrlWithGcsResolutionOnly => - marthaResponse.copy( - gsUri = Option("gs://abc/foo-123/abc123")) + drsResolverResponse.copy(gsUri = Option("gs://abc/foo-123/abc123")) case MockDrsPaths.fakeDrsUrlWithoutAnyResolution => - marthaResponse + drsResolverResponse case MockDrsPaths.fakeDrsUrlWithAccessUrlResolutionOnly => - marthaResponse.copy( - accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None))) + drsResolverResponse.copy(accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None))) case MockDrsPaths.fakeDrsUrlWithAccessUrlAndGcsResolution => - marthaResponse.copy( - accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None)), - gsUri = Option("gs://some/uri")) + drsResolverResponse.copy(accessUrl = Option(AccessUrl(url = "http://abc/def/ghi.bam", headers = None)), + gsUri = Option("gs://some/uri") + ) case e => throw new RuntimeException(s"Unexpected exception in DRS localization test code: $e") } } @@ -341,5 +418,7 @@ class MockDrsLocalizerDrsPathResolver(drsConfig: DrsConfig) extends object MockDrsLocalizerDrsPathResolver { val FakeHashes: Option[Map[String, String]] = Option(Map("md5" -> "abc123", "crc32c" -> "34fd67")) - val FakeAccessTokenStrategy: AccessTokenStrategy = () => "testing code: do not call me".invalidNel + val FakeAccessTokenStrategy: DrsCredentials = new DrsCredentials { + override def getAccessToken: ErrorOr[String] = "testing code: do not call me".invalidNel + } } diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala deleted file mode 100644 index df7512dd81a..00000000000 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/AccessUrlDownloaderSpec.scala +++ /dev/null @@ -1,59 +0,0 @@ -package drs.localizer.downloaders - -import cats.effect.ExitCode -import cats.syntax.validated._ -import cloud.nio.impl.drs.AccessUrl -import common.assertion.CromwellTimeoutSpec -import org.scalatest.flatspec.AnyFlatSpec -import org.scalatest.matchers.should.Matchers -import org.scalatest.prop.TableDrivenPropertyChecks._ - -class AccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { - it should "return the correct download script for a url-only access URL, no requester pays" in { - val fakeDownloadLocation = "/root/foo/foo-123.bam" - val fakeAccessUrl = "http://abc/def/ghi.bam" - - val downloader = AccessUrlDownloader( - accessUrl = AccessUrl(url = fakeAccessUrl, headers = None), - downloadLoc = fakeDownloadLocation, - hashes = None - ) - - val expected = s"""mkdir -p $$(dirname '$fakeDownloadLocation') && rm -f '$fakeDownloadLocation' && getm --checksum-algorithm 'null' --checksum null --filepath '$fakeDownloadLocation' '$fakeAccessUrl'""".validNel - - downloader.generateDownloadScript shouldBe expected - } - - { - val results = Table( - ("exitCode", "stderr", "download result"), - (0, "", DownloadSuccess), - // In `getm` version 0.0.4 checksum failures currently exit 0. - (0, "oh me oh my: AssertionError: Checksum failed!!!", ChecksumFailure), - // Unrecognized because of non-zero exit code without an HTTP status, despite what looks like a checksum failure. - (1, "oh me oh my: AssertionError: Checksum failed!!!", UnrecognizedRetryableDownloadFailure(ExitCode(1))), - // Unrecognized because of zero exit status with stderr that does not look like a checksum failure. - (0, "what the", UnrecognizedRetryableDownloadFailure(ExitCode(0))), - // Unrecognized because of non-zero exit code without an HTTP status. - (1, " foobar ", UnrecognizedRetryableDownloadFailure(ExitCode(1))), - // Unrecognized because of zero exit status with stderr that does not look like a checksum failure. - (0, """ERROR:getm.cli possibly some words "status_code": 503 words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))), - // Recognized because of non-zero exit status and an HTTP status. - (1, """ERROR:getm.cli possibly some words "status_code": 503 words""", RecognizedRetryableDownloadFailure(ExitCode(1))), - // Recognized because of non-zero exit status and an HTTP status. - (1, """ERROR:getm.cli possibly some words "status_code": 408 more words""", RecognizedRetryableDownloadFailure(ExitCode(1))), - // Recognized and non-retryable because of non-zero exit status and 404 HTTP status. - (1, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", FatalDownloadFailure(ExitCode(1))), - // Unrecognized because of zero exit status and 404 HTTP status. - (0, """ERROR:getm.cli possibly some words "status_code": 404 even more words""", UnrecognizedRetryableDownloadFailure(ExitCode(0))), - ) - - val accessUrlDownloader = AccessUrlDownloader(null, null, null) - - forAll(results) { (exitCode, stderr, expected) => - it should s"produce $expected for exitCode $exitCode and stderr '$stderr'" in { - accessUrlDownloader.toDownloadResult(GetmResult(exitCode, stderr)) shouldBe expected - } - } - } -} diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala new file mode 100644 index 00000000000..6eae90aa842 --- /dev/null +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/BulkAccessUrlDownloaderSpec.scala @@ -0,0 +1,133 @@ +package drs.localizer.downloaders + +import cats.effect.{ExitCode, IO} +import cloud.nio.impl.drs.{AccessUrl, DrsResolverResponse} +import common.assertion.CromwellTimeoutSpec +import org.scalatest.prop.TableDrivenPropertyChecks._ +import drs.localizer.{ResolvedDrsUrl, URIType} + +import org.scalatest.flatspec.AnyFlatSpec +import org.scalatest.matchers.should.Matchers + +import java.nio.file.Path + +class BulkAccessUrlDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matchers { + val ex1 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url123", None))), + "path/to/local/download/dest", + URIType.ACCESS + ) + val ex2 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1234", None))), + "path/to/local/download/dest2", + URIType.ACCESS + ) + val ex3 = ResolvedDrsUrl(DrsResolverResponse(accessUrl = Option(AccessUrl("https://my.fake/url1235", None))), + "path/to/local/download/dest3", + URIType.ACCESS + ) + val emptyList: List[ResolvedDrsUrl] = List() + val oneElement: List[ResolvedDrsUrl] = List(ex1) + val threeElements: List[ResolvedDrsUrl] = List(ex1, ex2, ex3) + + it should "correctly parse a collection of Access Urls into a manifest.json" in { + val expected = + """[{ + | "url": "https://my.fake/url123", + | "filepath": "path/to/local/download/dest" + |}, { + | "url": "https://my.fake/url1234", + | "filepath": "path/to/local/download/dest2" + |}, { + | "url": "https://my.fake/url1235", + | "filepath": "path/to/local/download/dest3" + |}]""".stripMargin + val downloader = BulkAccessUrlDownloader(threeElements) + + val filepath: IO[Path] = downloader.generateJsonManifest(threeElements) + val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString) + val lines = + try source.mkString + finally source.close() + lines shouldBe expected + } + + it should "properly construct empty JSON array from empty list." in { + val expected: String = "[]" + val downloader = BulkAccessUrlDownloader(emptyList) + val filepath: IO[Path] = downloader.generateJsonManifest(emptyList) + val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString) + val lines = + try source.mkString + finally source.close() + lines shouldBe expected + } + + it should "properly construct JSON array from single element list." in { + val expected: String = + s"""|[{ + | "url": "https://my.fake/url123", + | "filepath": "path/to/local/download/dest" + |}]""".stripMargin + + val downloader = BulkAccessUrlDownloader(oneElement) + val filepath: IO[Path] = downloader.generateJsonManifest(oneElement) + val source = scala.io.Source.fromFile(filepath.unsafeRunSync().toString) + val lines = + try source.mkString + finally source.close() + lines shouldBe expected + } + + it should "properly construct the invocation command" in { + val downloader = BulkAccessUrlDownloader(oneElement) + val filepath: Path = downloader.generateJsonManifest(threeElements).unsafeRunSync() + val expected = s"""timeout 24h getm --manifest ${filepath.toString} -vv""" + downloader.generateGetmCommand(filepath) shouldBe expected + } + + { + val results = Table( + ("exitCode", "stderr", "download result"), + (0, "", DownloadSuccess), + // In `getm` version 0.0.4 checksum failures currently exit 0. + (0, "oh me oh my: AssertionError: Checksum failed!!!", ChecksumFailure), + // Unrecognized because of non-zero exit code without an HTTP status, despite what looks like a checksum failure. + (1, "oh me oh my: AssertionError: Checksum failed!!!", UnrecognizedRetryableDownloadFailure(ExitCode(1))), + // Unrecognized because of zero exit status with stderr that does not look like a checksum failure. + (0, "what the", UnrecognizedRetryableDownloadFailure(ExitCode(0))), + // Unrecognized because of non-zero exit code without an HTTP status. + (1, " foobar ", UnrecognizedRetryableDownloadFailure(ExitCode(1))), + // Unrecognized because of zero exit status with stderr that does not look like a checksum failure. + (0, + """ERROR:getm.cli possibly some words "status_code": 503 words""", + UnrecognizedRetryableDownloadFailure(ExitCode(0)) + ), + // Recognized because of non-zero exit status and an HTTP status. + (1, + """ERROR:getm.cli possibly some words "status_code": 503 words""", + RecognizedRetryableDownloadFailure(ExitCode(1)) + ), + // Recognized because of non-zero exit status and an HTTP status. + (1, + """ERROR:getm.cli possibly some words "status_code": 408 more words""", + RecognizedRetryableDownloadFailure(ExitCode(1)) + ), + // Recognized and non-retryable because of non-zero exit status and 404 HTTP status. + (1, + """ERROR:getm.cli possibly some words "status_code": 404 even more words""", + FatalDownloadFailure(ExitCode(1)) + ), + // Unrecognized because of zero exit status and 404 HTTP status. + (0, + """ERROR:getm.cli possibly some words "status_code": 404 even more words""", + UnrecognizedRetryableDownloadFailure(ExitCode(0)) + ) + ) + val bulkDownloader = BulkAccessUrlDownloader(null) + + forAll(results) { (exitCode, stderr, expected) => + it should s"produce $expected for exitCode $exitCode and stderr '$stderr'" in { + bulkDownloader.toDownloadResult(GetmResult(exitCode, stderr)) shouldBe expected + } + } + } +} diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GcsUriDownloaderSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GcsUriDownloaderSpec.scala index 66c4ff6fabd..07eb5ede181 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GcsUriDownloaderSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GcsUriDownloaderSpec.scala @@ -11,7 +11,7 @@ class GcsUriDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat val fakeDownloadLocation = "/root/foo/foo-123.bam" val fakeRequesterPaysId = "fake-billing-project" - it should "return correct download script for a drs url without Requester Pays ID and Google SA returned from Martha" in { + it should "return correct download script for a drs url without Requester Pays ID and Google SA returned from the DRS Resolver" in { val gcsUrl = "gs://foo/bar.bam" val downloader = new GcsUriDownloader( gcsUrl = gcsUrl, @@ -44,7 +44,7 @@ class GcsUriDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat downloader.generateDownloadScript(gcsUrl = gcsUrl, saJsonPathOption = None) shouldBe expectedDownloadScript } - it should "inject Requester Pays flag & gcloud auth using SA returned from Martha" in { + it should "inject Requester Pays flag & gcloud auth using SA returned from the DRS Resolver" in { val gcsUrl = "gs://foo/bar.bam" val downloader = new GcsUriDownloader( gcsUrl = gcsUrl, @@ -60,11 +60,11 @@ class GcsUriDownloaderSpec extends AnyFlatSpec with CromwellTimeoutSpec with Mat s"""set -euo pipefail |set +e | - |# Set gsutil to use the service account returned from Martha + |# Set gsutil to use the service account returned from the DRS Resolver |gcloud auth activate-service-account --key-file=${fakeSAJsonPath.toString} > gcloud_output.txt 2>&1 |RC_GCLOUD=$$? |if [ "$$RC_GCLOUD" != "0" ]; then - | echo "Failed to activate service account returned from Martha. File won't be downloaded. Error: $$(cat gcloud_output.txt)" >&2 + | echo "Failed to activate service account returned from the DRS Resolver. File won't be downloaded. Error: $$(cat gcloud_output.txt)" >&2 | exit "$$RC_GCLOUD" |else | echo "Successfully activated service account; Will continue with download. $$(cat gcloud_output.txt)" diff --git a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala index 24ea087cb8f..a8ac76fa6c0 100644 --- a/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala +++ b/cromwell-drs-localizer/src/test/scala/drs/localizer/downloaders/GetmChecksumSpec.scala @@ -17,9 +17,15 @@ class GetmChecksumSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher (Option(Map("something weird" -> "012345", "md5" -> "abcdefg")), "https://whatever", Md5("abcdefg")), (Option(Map("something weird" -> "abcdefg", "crc32c" -> "012345")), "https://whatever", Crc32c("012345")), (Option(Map("etag" -> "abcdefg", "crc32c" -> "012345")), "https://whatever", Crc32c("012345")), - (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), "https://whatever", Unsupported("etag, something weird")), - (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), "https://whatever.s3.amazonaws.com/foo", AwsEtag("abcdefg")), - (None, "https://whatever.s3.amazonaws.com/foo", Null), + (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), + "https://whatever", + Unsupported("etag, something weird") + ), + (Option(Map("etag" -> "abcdefg", "something weird" -> "012345")), + "https://whatever.s3.amazonaws.com/foo", + AwsEtag("abcdefg") + ), + (None, "https://whatever.s3.amazonaws.com/foo", Null) ) forAll(results) { (hashes, url, expected) => @@ -31,13 +37,23 @@ class GetmChecksumSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher val results = Table( ("description", "algorithm", "expected"), ("md5 hex", Md5("abcdef"), "--checksum-algorithm 'md5' --checksum abcdef".validNel), - ("md5 base64", Md5("cR84lXY1y17c3q7/7riLEA=="), "--checksum-algorithm 'md5' --checksum 711f38957635cb5edcdeaeffeeb88b10".validNel), - ("md5 gibberish", Md5("what is this???"), "Invalid md5 checksum value is neither hex nor base64: what is this???".invalidNel), - ("crc32c", Crc32c("012345"), "--checksum-algorithm 'gs_crc32c' --checksum 012345".validNel), + ("md5 base64", + Md5("cR84lXY1y17c3q7/7riLEA=="), + "Invalid checksum value, expected hex but got: cR84lXY1y17c3q7/7riLEA==".invalidNel + ), + ("md5 gibberish", + Md5("what is this???"), + "Invalid checksum value, expected hex but got: what is this???".invalidNel + ), + ("crc32c", Crc32c("012345"), "--checksum-algorithm 'gs_crc32c' --checksum ASNF".validNel), + ("crc32c gibberish", Crc32c("????"), "Invalid checksum value, expected hex but got: ????".invalidNel), ("AWS ETag", AwsEtag("012345"), "--checksum-algorithm 's3_etag' --checksum 012345".validNel), // Escape checksum values constructed from unvalidated data returned by DRS servers. - ("Unsupported", Unsupported("Robert'); DROP TABLE Students;\n --\\"), raw"--checksum-algorithm 'null' --checksum Robert\'\)\;\ DROP\ TABLE\ Students\;\ --\\".validNel), - ("Null", Null, "--checksum-algorithm 'null' --checksum null".validNel), + ("Unsupported", + Unsupported("Robert'); DROP TABLE Students;\n --\\"), + raw"--checksum-algorithm 'null' --checksum Robert\'\)\;\ DROP\ TABLE\ Students\;\ --\\".validNel + ), + ("Null", Null, "--checksum-algorithm 'null' --checksum null".validNel) ) forAll(results) { (description, algorithm, expected) => @@ -46,4 +62,19 @@ class GetmChecksumSpec extends AnyFlatSpec with CromwellTimeoutSpec with Matcher } } } + + it should "correctly validate hex strings" in { + val results = Table( + ("test string", "expected"), + ("", "Invalid checksum value, expected hex but got: ".invalidNel), + (" ", "Invalid checksum value, expected hex but got: ".invalidNel), + ("myfavoritestring", "Invalid checksum value, expected hex but got: myfavoritestring".invalidNel), + (" AbC123 ", "AbC123".validNel), + ("456", "456".validNel) + ) + + forAll(results) { (testString, expected) => + GetmChecksum.validateHex(testString) shouldBe expected + } + } } diff --git a/cromwell.example.backends/GCPBATCH.conf b/cromwell.example.backends/GCPBATCH.conf new file mode 100644 index 00000000000..ba554e3322d --- /dev/null +++ b/cromwell.example.backends/GCPBATCH.conf @@ -0,0 +1,104 @@ +# This is an example of how you can use the Google Cloud Batch backend +# provider. *This is not a complete configuration file!* The +# content here should be copy pasted into the backend -> providers section +# of cromwell.example.backends/cromwell.examples.conf in the root of the repository. +# You should uncomment lines that you want to define, and read carefully to customize +# the file. + +# Documentation +# https://cromwell.readthedocs.io/en/stable/backends/Google/ + +backend { + default = GCPBATCH + + providers { + GCPBATCH { + actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory" + config { + # Google project + project = "my-cromwell-workflows" + + # Base bucket for workflow executions + root = "gs://my-cromwell-workflows-bucket" + + # Polling for completion backs-off gradually for slower-running jobs. + # This is the maximum polling interval (in seconds): + maximum-polling-interval = 600 + + # Optional Dockerhub Credentials. Can be used to access private docker images. + dockerhub { + # account = "" + # token = "" + } + + # Optional configuration to use high security network (Virtual Private Cloud) for running jobs. + # See https://cromwell.readthedocs.io/en/stable/backends/Google/ for more details. + # virtual-private-cloud { + # network-label-key = "network-key" + # auth = "application-default" + # } + + # Global pipeline timeout + # Defaults to 7 days; max 30 days + # batch-timeout = 7 days + + genomics { + # A reference to an auth defined in the `google` stanza at the top. This auth is used to create + # Batch Jobs and manipulate auth JSONs. + auth = "application-default" + + + // alternative service account to use on the launched compute instance + // NOTE: If combined with service account authorization, both that service account and this service account + // must be able to read and write to the 'root' GCS path + compute-service-account = "default" + + # Location to submit jobs to Batch and store job metadata. + location = "us-central1" + + # Specifies the minimum file size for `gsutil cp` to use parallel composite uploads during delocalization. + # Parallel composite uploads can result in a significant improvement in delocalization speed for large files + # but may introduce complexities in downloading such files from GCS, please see + # https://cloud.google.com/storage/docs/gsutil/commands/cp#parallel-composite-uploads for more information. + # + # If set to 0 parallel composite uploads are turned off. The default Cromwell configuration turns off + # parallel composite uploads, this sample configuration turns it on for files of 150M or larger. + parallel-composite-upload-threshold="150M" + } + + filesystems { + gcs { + # A reference to a potentially different auth for manipulating files via engine functions. + auth = "application-default" + # Google project which will be billed for the requests + project = "google-billing-project" + + caching { + # When a cache hit is found, the following duplication strategy will be followed to use the cached outputs + # Possible values: "copy", "reference". Defaults to "copy" + # "copy": Copy the output files + # "reference": DO NOT copy the output files but point to the original output files instead. + # Will still make sure than all the original output files exist and are accessible before + # going forward with the cache hit. + duplication-strategy = "copy" + } + } + } + + default-runtime-attributes { + cpu: 1 + failOnStderr: false + continueOnReturnCode: 0 + memory: "2048 MB" + bootDiskSizeGb: 10 + # Allowed to be a String, or a list of Strings + disks: "local-disk 10 SSD" + noAddress: false + preemptible: 0 + zones: ["us-central1-a", "us-central1-b"] + } + + } + } + } +} diff --git a/cromwell.example.backends/TES.conf b/cromwell.example.backends/TES.conf index a0bfb7cb141..509cd0d5d90 100644 --- a/cromwell.example.backends/TES.conf +++ b/cromwell.example.backends/TES.conf @@ -28,6 +28,22 @@ backend { disk: "2 GB" preemptible: false } + + # Backoff behavior for task status polling and execution retries are configurable, with defaults + # shown below. All four fields must be set for each backoff if overriding. + # + # poll-backoff { + # min: "10 seconds" + # max: "5 minutes" + # multiplier: 1.1 + # randomization-factor: 0.5 + # } + # execute-or-recover-backoff { + # min: "3 seconds" + # max: "30 seconds" + # multiplier: 1.1 + # randomization-factor: 0.5 + # } } } } diff --git a/cromwell.example.backends/cromwell.examples.conf b/cromwell.example.backends/cromwell.examples.conf index 49b676a2c7c..00f99d2ff5a 100644 --- a/cromwell.example.backends/cromwell.examples.conf +++ b/cromwell.example.backends/cromwell.examples.conf @@ -308,17 +308,6 @@ languages { } } } - CWL { - versions { - "v1.0" { - # language-factory = "languages.cwl.CwlV1_0LanguageFactory" - # config { - # strict-validation: false - # enabled: true - # } - } - } - } } # Here is where you can define the backend providers that Cromwell understands. @@ -509,6 +498,11 @@ services { # # count against this limit. # metadata-read-row-number-safety-threshold = 1000000 # + # # Remove any UTF-8 mb4 (4 byte) characters from metadata keys in the list. + # # These characters (namely emojis) will cause metadata writing to fail in database collations + # # that do not support 4 byte UTF-8 characters. + # metadata-keys-to-sanitize-utf8mb4 = ["submittedFiles:workflow", "commandLine"] + # # metadata-write-statistics { # # Not strictly necessary since the 'metadata-write-statistics' section itself is enough for statistics to be recorded. # # However, this can be set to 'false' to disable statistics collection without deleting the section. diff --git a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala index b2a088e681b..18df2ff5244 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/CromwellClient.scala @@ -20,10 +20,10 @@ import scala.concurrent.ExecutionContext class CromwellClient(val cromwellUrl: URL, val apiVersion: String, - val defaultCredentials: Option[HttpCredentials]=None) - (implicit actorSystem: ActorSystem, materializer: ActorMaterializer) { + val defaultCredentials: Option[HttpCredentials] = None +)(implicit actorSystem: ActorSystem, materializer: ActorMaterializer) { - lazy val defaultAuthorization: Option[Authorization] = defaultCredentials.map { Authorization(_) } + lazy val defaultAuthorization: Option[Authorization] = defaultCredentials.map(Authorization(_)) lazy val defaultHeaders: List[HttpHeader] = defaultAuthorization.toList lazy val engineEndpoint = s"$cromwellUrl/engine/$apiVersion" @@ -33,7 +33,7 @@ class CromwellClient(val cromwellUrl: URL, lazy val submitEndpoint = workflowsEndpoint lazy val batchSubmitEndpoint = s"$submitEndpoint/batch" - def describeEndpoint= s"$womtoolEndpoint/describe" + def describeEndpoint = s"$womtoolEndpoint/describe" def queryEndpoint(args: List[(String, String)]): Uri = { val base = s"$workflowsEndpoint/query" @@ -49,11 +49,20 @@ class CromwellClient(val cromwellUrl: URL, def abortEndpoint(workflowId: WorkflowId): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "abort") def statusEndpoint(workflowId: WorkflowId): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "status") - def metadataEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "metadata", args) - def outputsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "outputs", args) + def metadataEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = + workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "metadata", args) + def outputsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = + workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "outputs", args) def labelsEndpoint(workflowId: WorkflowId): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "labels") - def logsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "logs", args) - def diffEndpoint(workflowA: WorkflowId, callA: String, indexA: ShardIndex, workflowB: WorkflowId, callB: String, indexB: ShardIndex): String = { + def logsEndpoint(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None): Uri = + workflowSpecificGetEndpoint(workflowsEndpoint, workflowId, "logs", args) + def diffEndpoint(workflowA: WorkflowId, + callA: String, + indexA: ShardIndex, + workflowB: WorkflowId, + callB: String, + indexB: ShardIndex + ): String = { def shardParam(aOrB: String, s: ShardIndex) = s.index.map(i => s"&index$aOrB=$i.toString").getOrElse("") s"$workflowsEndpoint/callcaching/diff?workflowA=$workflowA&callA=$callA&workflowB=$workflowB&callB=$callB${shardParam("A", indexA)}${shardParam("B", indexB)}" } @@ -69,40 +78,48 @@ class CromwellClient(val cromwellUrl: URL, import model.WorkflowDescriptionJsonSupport._ import model.CromwellQueryResultJsonSupport._ - def submit(workflow: WorkflowSubmission) - (implicit ec: ExecutionContext): FailureResponseOrT[SubmittedWorkflow] = { + def submit(workflow: WorkflowSubmission)(implicit ec: ExecutionContext): FailureResponseOrT[SubmittedWorkflow] = { val requestEntity = requestEntityForSubmit(workflow) - makeRequest[CromwellStatus](HttpRequest(HttpMethods.POST, submitEndpoint, List.empty[HttpHeader], requestEntity)) map { status => + makeRequest[CromwellStatus]( + HttpRequest(HttpMethods.POST, submitEndpoint, List.empty[HttpHeader], requestEntity) + ) map { status => SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, workflow) } } - def describe(workflow: WorkflowDescribeRequest) - (implicit ec: ExecutionContext): FailureResponseOrT[WaasDescription] = { + def describe( + workflow: WorkflowDescribeRequest + )(implicit ec: ExecutionContext): FailureResponseOrT[WaasDescription] = { val requestEntity = requestEntityForDescribe(workflow) makeRequest[WaasDescription](HttpRequest(HttpMethods.POST, describeEndpoint, List.empty[HttpHeader], requestEntity)) } - def submitBatch(workflow: WorkflowBatchSubmission) - (implicit ec: ExecutionContext): FailureResponseOrT[List[SubmittedWorkflow]] = { + def submitBatch( + workflow: WorkflowBatchSubmission + )(implicit ec: ExecutionContext): FailureResponseOrT[List[SubmittedWorkflow]] = { import DefaultJsonProtocol._ val requestEntity = requestEntityForSubmit(workflow) // Make a set of submissions that represent the batch (so we can zip with the results later): - val submissionSet = workflow.inputsBatch.map(inputs => WorkflowSingleSubmission( - workflowSource = workflow.workflowSource, - workflowUrl = workflow.workflowUrl, - workflowRoot = workflow.workflowRoot, - workflowType = workflow.workflowType, - workflowTypeVersion = workflow.workflowTypeVersion, - inputsJson = Option(inputs), - options = workflow.options, - labels = workflow.labels, - zippedImports = workflow.zippedImports)) - - makeRequest[List[CromwellStatus]](HttpRequest(HttpMethods.POST, batchSubmitEndpoint, List.empty[HttpHeader], requestEntity)) map { statuses => + val submissionSet = workflow.inputsBatch.map(inputs => + WorkflowSingleSubmission( + workflowSource = workflow.workflowSource, + workflowUrl = workflow.workflowUrl, + workflowRoot = workflow.workflowRoot, + workflowType = workflow.workflowType, + workflowTypeVersion = workflow.workflowTypeVersion, + inputsJson = Option(inputs), + options = workflow.options, + labels = workflow.labels, + zippedImports = workflow.zippedImports + ) + ) + + makeRequest[List[CromwellStatus]]( + HttpRequest(HttpMethods.POST, batchSubmitEndpoint, List.empty[HttpHeader], requestEntity) + ) map { statuses => val zipped = submissionSet.zip(statuses) zipped map { case (submission, status) => SubmittedWorkflow(WorkflowId.fromString(status.id), cromwellUrl, submission) @@ -110,48 +127,42 @@ class CromwellClient(val cromwellUrl: URL, } } - def abort(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = { + def abort(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = simpleRequest[CromwellStatus](uri = abortEndpoint(workflowId), method = HttpMethods.POST) map WorkflowStatus.apply - } - def status(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = { + def status(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowStatus] = simpleRequest[CromwellStatus](statusEndpoint(workflowId)) map WorkflowStatus.apply - } def metadata(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None, headers: List[HttpHeader] = defaultHeaders - )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = { - simpleRequest[String](metadataEndpoint(workflowId, args), headers=headers) map WorkflowMetadata - } + )(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = + simpleRequest[String](metadataEndpoint(workflowId, args), headers = headers) map WorkflowMetadata - def outputs(workflowId: WorkflowId, - args: Option[Map[String, List[String]]] = None)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowOutputs] = { + def outputs(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowOutputs] = simpleRequest[WorkflowOutputs](outputsEndpoint(workflowId, args)) - } - def labels(workflowId: WorkflowId, - headers: List[HttpHeader] = defaultHeaders) - (implicit ec: ExecutionContext): FailureResponseOrT[WorkflowLabels] = { + def labels(workflowId: WorkflowId, headers: List[HttpHeader] = defaultHeaders)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowLabels] = simpleRequest[WorkflowLabels](labelsEndpoint(workflowId), headers = headers) - } - def addLabels(workflowId: WorkflowId, - newLabels: List[Label], - headers: List[HttpHeader] = defaultHeaders) - (implicit ec: ExecutionContext): FailureResponseOrT[WorkflowLabels] = { + def addLabels(workflowId: WorkflowId, newLabels: List[Label], headers: List[HttpHeader] = defaultHeaders)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowLabels] = { val requestEntity = requestEntityForAddLabels(newLabels) makeRequest[WorkflowLabels](HttpRequest(HttpMethods.PATCH, labelsEndpoint(workflowId), headers, requestEntity)) } - def logs(workflowId: WorkflowId, - args: Option[Map[String, List[String]]] = None)(implicit ec: ExecutionContext): FailureResponseOrT[WorkflowMetadata] = { + def logs(workflowId: WorkflowId, args: Option[Map[String, List[String]]] = None)(implicit + ec: ExecutionContext + ): FailureResponseOrT[WorkflowMetadata] = simpleRequest[String](logsEndpoint(workflowId, args)) map WorkflowMetadata - } - def query(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[CromwellQueryResults] = { + def query(workflowId: WorkflowId)(implicit ec: ExecutionContext): FailureResponseOrT[CromwellQueryResults] = simpleRequest[CromwellQueryResults](queryEndpoint(List(("id", workflowId.id.toString)))) - } def callCacheDiff(workflowA: WorkflowId, callA: String, @@ -159,27 +170,26 @@ class CromwellClient(val cromwellUrl: URL, workflowB: WorkflowId, callB: String, shardIndexB: ShardIndex - )(implicit ec: ExecutionContext): FailureResponseOrT[CallCacheDiff] = { + )(implicit ec: ExecutionContext): FailureResponseOrT[CallCacheDiff] = simpleRequest[CallCacheDiff](diffEndpoint(workflowA, callA, shardIndexA, workflowB, callB, shardIndexB)) - } - def backends(implicit ec: ExecutionContext): FailureResponseOrT[CromwellBackends] = { + def backends(implicit ec: ExecutionContext): FailureResponseOrT[CromwellBackends] = simpleRequest[CromwellBackends](backendsEndpoint) - } - def version(implicit ec: ExecutionContext): FailureResponseOrT[CromwellVersion] = { + def version(implicit ec: ExecutionContext): FailureResponseOrT[CromwellVersion] = simpleRequest[CromwellVersion](versionEndpoint) - } - private [api] def executeRequest(request: HttpRequest, headers: List[HttpHeader]) = Http().singleRequest(request.withHeaders(headers)) + private[api] def executeRequest(request: HttpRequest, headers: List[HttpHeader]) = + Http().singleRequest(request.withHeaders(headers)) /** * * @tparam A The type of response expected. Must be supported by an implicit unmarshaller from ResponseEntity. */ - private def makeRequest[A](request: HttpRequest, headers: List[HttpHeader] = defaultHeaders) - (implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): - FailureResponseOrT[A] = { + private def makeRequest[A](request: HttpRequest, headers: List[HttpHeader] = defaultHeaders)(implicit + um: Unmarshaller[ResponseEntity, A], + ec: ExecutionContext + ): FailureResponseOrT[A] = { implicit def cs = IO.contextShift(ec) for { response <- executeRequest(request, headers).asFailureResponseOrT @@ -191,11 +201,9 @@ class CromwellClient(val cromwellUrl: URL, private def simpleRequest[A](uri: Uri, method: HttpMethod = HttpMethods.GET, - headers: List[HttpHeader] = defaultHeaders) - (implicit um: Unmarshaller[ResponseEntity, A], - ec: ExecutionContext): FailureResponseOrT[A] = { + headers: List[HttpHeader] = defaultHeaders + )(implicit um: Unmarshaller[ResponseEntity, A], ec: ExecutionContext): FailureResponseOrT[A] = makeRequest[A](HttpRequest(uri = uri, method = method), headers) - } private val decoders = Map( HttpEncodings.gzip -> Gzip, @@ -203,15 +211,14 @@ class CromwellClient(val cromwellUrl: URL, HttpEncodings.identity -> NoCoding ) - private def decodeResponse(response: HttpResponse): IO[HttpResponse] = { + private def decodeResponse(response: HttpResponse): IO[HttpResponse] = decoders.get(response.encoding) map { decoder => IO(decoder.decodeMessage(response)) } getOrElse IO.raiseError(UnsuccessfulRequestException(s"No decoder for ${response.encoding}", response)) - } } object CromwellClient { - final implicit class EnhancedHttpResponse(val response: HttpResponse) extends AnyVal { + implicit final class EnhancedHttpResponse(val response: HttpResponse) extends AnyVal { def toEntity: IO[Unmarshal[ResponseEntity]] = response match { case HttpResponse(_: StatusCodes.Success, _, entity, _) => IO(Unmarshal(entity)) @@ -233,18 +240,17 @@ object CromwellClient { "workflowInputs" -> workflowSubmission.inputsJson, "workflowOptions" -> workflowSubmission.options, "labels" -> workflowSubmission.labels.map(_.toJson.toString) - ) collect { - case (name, Some(source: String)) => - Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) + ) collect { case (name, Some(source: String)) => + Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) } val zipBodyParts = Map( "workflowDependencies" -> workflowSubmission.zippedImports - ) collect { - case (name, Some(file)) => Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) + ) collect { case (name, Some(file)) => + Multipart.FormData.BodyPart.fromPath(name, MediaTypes.`application/zip`, file.path) } - val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq : _*) + val multipartFormData = Multipart.FormData((sourceBodyParts ++ zipBodyParts).toSeq: _*) multipartFormData.toEntity() } @@ -256,12 +262,11 @@ object CromwellClient { "workflowType" -> describeRequest.workflowType, "workflowTypeVersion" -> describeRequest.workflowTypeVersion, "workflowInputs" -> describeRequest.inputsJson - ) collect { - case (name, Some(source: String)) => - Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) + ) collect { case (name, Some(source: String)) => + Multipart.FormData.BodyPart(name, HttpEntity(MediaTypes.`application/json`, ByteString(source))) } - val multipartFormData = Multipart.FormData(sourceBodyParts.toSeq : _*) + val multipartFormData = Multipart.FormData(sourceBodyParts.toSeq: _*) multipartFormData.toEntity() } @@ -273,12 +278,16 @@ object CromwellClient { /** * @param args an optional map of HTTP arguments which will be added to the URL */ - private [api] def workflowSpecificGetEndpoint(submitEndpoint: String, workflowId: WorkflowId, endpoint: String, args: Option[Map[String, List[String]]] = None) = { + private[api] def workflowSpecificGetEndpoint(submitEndpoint: String, + workflowId: WorkflowId, + endpoint: String, + args: Option[Map[String, List[String]]] = None + ) = { val url = s"$submitEndpoint/$workflowId/$endpoint" val queryBuilder = Uri.Query.newBuilder - args.getOrElse(Map.empty).foreach({ - case (key, l) => l.foreach(v => queryBuilder.+=(key -> v)) - }) + args.getOrElse(Map.empty).foreach { case (key, l) => + l.foreach(v => queryBuilder.+=(key -> v)) + } val queryResult = queryBuilder.result() Uri(url).withQuery(queryResult) } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala index fa4e7fb9187..64cafc2b611 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CallCacheDiff.scala @@ -4,9 +4,17 @@ import ShardIndexFormatter._ import WorkflowIdJsonFormatter._ import spray.json.DefaultJsonProtocol -case class CallCacheDiffCallDescription(executionStatus: String, allowResultReuse: Boolean, callFqn: String, jobIndex: ShardIndex, workflowId: WorkflowId) +case class CallCacheDiffCallDescription(executionStatus: String, + allowResultReuse: Boolean, + callFqn: String, + jobIndex: ShardIndex, + workflowId: WorkflowId +) case class HashDifference(hashKey: String, callA: Option[String], callB: Option[String]) -case class CallCacheDiff(callA: CallCacheDiffCallDescription, callB: CallCacheDiffCallDescription, hashDifferential: List[HashDifference]) +case class CallCacheDiff(callA: CallCacheDiffCallDescription, + callB: CallCacheDiffCallDescription, + hashDifferential: List[HashDifference] +) object CallCacheDiffJsonSupport extends DefaultJsonProtocol { implicit val CallCacheDiffCallDescriptionFormat = jsonFormat5(CallCacheDiffCallDescription) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala index 4a54cb58469..658ea72489f 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/CromwellQueryResult.scala @@ -7,7 +7,13 @@ import cromwell.api.model.WorkflowStatusJsonFormatter._ case class CromwellQueryResults(results: Seq[CromwellQueryResult]) -case class CromwellQueryResult(name: Option[String], id: WorkflowId, status: WorkflowStatus, end: Option[OffsetDateTime], start: Option[OffsetDateTime], metadataArchiveStatus: String) +case class CromwellQueryResult(name: Option[String], + id: WorkflowId, + status: WorkflowStatus, + end: Option[OffsetDateTime], + start: Option[OffsetDateTime], + metadataArchiveStatus: String +) object CromwellQueryResultJsonSupport extends DefaultJsonProtocol { implicit val CromwellQueryResultJsonFormat = jsonFormat6(CromwellQueryResult) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala index fd9d88d2177..0d111626f84 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/Label.scala @@ -5,7 +5,7 @@ import scala.language.postfixOps object LabelsJsonFormatter extends DefaultJsonProtocol { implicit object LabelJsonFormat extends RootJsonFormat[List[Label]] { - def write(l: List[Label]) = JsObject(l map { label => label.key -> JsString(label.value)} :_* ) + def write(l: List[Label]) = JsObject(l map { label => label.key -> JsString(label.value) }: _*) def read(value: JsValue) = value.asJsObject.fields map { case (k, JsString(v)) => Label(k, v) case other => throw new UnsupportedOperationException(s"Cannot deserialize $other to a Label") diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala index 1688da92ff7..0db0f1c8f66 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/TimeUtil.scala @@ -4,6 +4,7 @@ import java.time.format.DateTimeFormatter import java.time.{OffsetDateTime, ZoneOffset} object TimeUtil { + /** * Instead of "one of" the valid ISO-8601 formats, standardize on this one: * https://github.com/openjdk/jdk/blob/jdk8-b120/jdk/src/share/classes/java/time/OffsetDateTime.java#L1886 @@ -11,12 +12,15 @@ object TimeUtil { private val Iso8601MillisecondsFormat = DateTimeFormatter.ofPattern("uuuu-MM-dd'T'HH:mm:ss.SSSXXXXX") implicit class EnhancedOffsetDateTime(val offsetDateTime: OffsetDateTime) extends AnyVal { + /** * Discards the original timezone and shifts the time to UTC, then returns the ISO-8601 formatted string with * exactly three digits of milliseconds. */ - def toUtcMilliString: String = Option(offsetDateTime).map( - _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) - ).orNull + def toUtcMilliString: String = Option(offsetDateTime) + .map( + _.atZoneSameInstant(ZoneOffset.UTC).format(Iso8601MillisecondsFormat) + ) + .orNull } } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala index e1f679a7b35..f83ca95fb9a 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WaasDescription.scala @@ -22,17 +22,20 @@ final case class WaasDescription(valid: Boolean, importedDescriptorTypes: List[WaasWorkflowDescriptorType], meta: JsObject, parameterMeta: JsObject, - isRunnableWorkflow: Boolean) + isRunnableWorkflow: Boolean +) final case class WaasDescriptionInputDefinition(name: String, valueType: WaasDescriptionWomType, optional: Option[Boolean], default: Option[JsValue], - typeDisplayName: String) + typeDisplayName: String +) final case class WaasDescriptionOutputDefinition(name: String, valueType: WaasDescriptionWomType, - typeDisplayName: String) + typeDisplayName: String +) final case class WaasDescriptionWomType(typeName: String) final case class WaasWorkflowDescriptorType(descriptorType: Option[String], descriptorTypeVersion: Option[String]) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala index 9ab7ea45ec4..1a46e2117f9 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowDescribeRequest.scala @@ -4,4 +4,5 @@ final case class WorkflowDescribeRequest(workflowSource: Option[String], workflowUrl: Option[String], workflowType: Option[String], workflowTypeVersion: Option[String], - inputsJson: Option[String]) + inputsJson: Option[String] +) diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala index f52495136c3..20c3a558790 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowId.scala @@ -30,4 +30,3 @@ object WorkflowIdJsonFormatter extends DefaultJsonProtocol { } } } - diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala index 6da1282d2da..70f9aa1efeb 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowStatus.scala @@ -25,7 +25,7 @@ case object Running extends NonTerminalStatus case object Aborting extends NonTerminalStatus object WorkflowStatus { - def apply(status: String): WorkflowStatus = { + def apply(status: String): WorkflowStatus = status match { case "Submitted" => Submitted case "Running" => Running @@ -35,7 +35,6 @@ object WorkflowStatus { case "Succeeded" => Succeeded case bad => throw new IllegalArgumentException(s"No such status: $bad") } - } def apply(workflowStatus: CromwellStatus): WorkflowStatus = apply(workflowStatus.status) } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala index 5f59368de13..e22d3e93e2e 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/WorkflowSubmission.scala @@ -22,7 +22,8 @@ final case class WorkflowSingleSubmission(workflowSource: Option[String], inputsJson: Option[String], options: Option[String], labels: Option[List[Label]], - zippedImports: Option[File]) extends WorkflowSubmission + zippedImports: Option[File] +) extends WorkflowSubmission final case class WorkflowBatchSubmission(workflowSource: Option[String], workflowUrl: Option[String], @@ -32,7 +33,8 @@ final case class WorkflowBatchSubmission(workflowSource: Option[String], inputsBatch: List[String], options: Option[String], labels: Option[List[Label]], - zippedImports: Option[File]) extends WorkflowSubmission { + zippedImports: Option[File] +) extends WorkflowSubmission { override val inputsJson: Option[String] = Option(inputsBatch.mkString(start = "[", sep = ",", end = "]")) } diff --git a/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala b/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala index 9067ce7157a..ccd75efac1f 100644 --- a/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala +++ b/cromwellApiClient/src/main/scala/cromwell/api/model/package.scala @@ -45,7 +45,7 @@ package object model { } implicit class EnhancedFailureResponseOrHttpResponseT(val responseIoT: FailureResponseOrT[HttpResponse]) - extends AnyVal { + extends AnyVal { def asHttpResponse: Future[HttpResponse] = { val io = responseIoT.value map { case Left(response) => response @@ -55,13 +55,14 @@ package object model { } } - implicit class EnhancedFailureResponseOrT[SuccessType](val responseIoT: FailureResponseOrT[SuccessType]) extends AnyVal { - final def timeout(duration: FiniteDuration) - (implicit timer: Timer[IO], cs: ContextShift[IO]): FailureResponseOrT[SuccessType] = { + implicit class EnhancedFailureResponseOrT[SuccessType](val responseIoT: FailureResponseOrT[SuccessType]) + extends AnyVal { + final def timeout( + duration: FiniteDuration + )(implicit timer: Timer[IO], cs: ContextShift[IO]): FailureResponseOrT[SuccessType] = EitherT(responseIoT.value.timeout(duration)) - } - def asIo(implicit materializer: ActorMaterializer, executionContext: ExecutionContext): IO[SuccessType] = { + def asIo(implicit materializer: ActorMaterializer, executionContext: ExecutionContext): IO[SuccessType] = responseIoT.value flatMap { case Left(response) => implicit def cs = IO.contextShift(executionContext) @@ -72,15 +73,13 @@ package object model { }) case Right(a) => IO.pure(a) } - } /** * Transforms the IO error from one type to another. */ def mapErrorWith(mapper: Throwable => IO[Nothing]): FailureResponseOrT[SuccessType] = { - def handleErrorIo[A](ioIn: IO[A]): IO[A] = { + def handleErrorIo[A](ioIn: IO[A]): IO[A] = ioIn handleErrorWith mapper - } responseIoT.mapK(FunctionK.lift(handleErrorIo)) } diff --git a/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala index 1bfecebc0d5..ab653523801 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/CromwellClientSpec.scala @@ -11,7 +11,6 @@ import org.scalatest.BeforeAndAfterAll import org.scalatest.flatspec.AsyncFlatSpec import org.scalatest.matchers.should.Matchers - class CromwellClientSpec extends AsyncFlatSpec with BeforeAndAfterAll with Matchers with TableDrivenPropertyChecks { behavior of "CromwellClient" @@ -40,86 +39,81 @@ class CromwellClientSpec extends AsyncFlatSpec with BeforeAndAfterAll with Match private val okRequestEntityTests = Table( ("description", "workflowSubmission", "expectedJsons", "expectedFiles"), - ("submit a wdl", - WorkflowSingleSubmission(Option("wdl"), None, None, None, None, None, None, None, None), - Map("workflowSource" -> "wdl"), - Map() + WorkflowSingleSubmission(Option("wdl"), None, None, None, None, None, None, None, None), + Map("workflowSource" -> "wdl"), + Map() ), - ("batch submit a wdl", - WorkflowBatchSubmission(Option("wdl"), None, None, None, None, List(), None, None, None), - Map("workflowSource" -> "wdl", "workflowInputs" -> "[]"), - Map() + WorkflowBatchSubmission(Option("wdl"), None, None, None, None, List(), None, None, None), + Map("workflowSource" -> "wdl", "workflowInputs" -> "[]"), + Map() ), - ("submit a wdl with data", - WorkflowSingleSubmission( - Option("wdl"), - None, - None, - Option("wfType"), - Option("wfTypeVersion"), - Option("inputsJson"), - Option("optionsJson"), - Option(List(Label("labelKey", "labelValue"))), - Option(tempFile) - ), - Map( - "workflowSource" -> "wdl", - "workflowType" -> "wfType", - "workflowTypeVersion" -> "wfTypeVersion", - "workflowInputs" -> "inputsJson", - "workflowOptions" -> "optionsJson", - "labels" -> """{"labelKey":"labelValue"}""" - ), - Map("workflowDependencies" -> tempFile) + WorkflowSingleSubmission( + Option("wdl"), + None, + None, + Option("wfType"), + Option("wfTypeVersion"), + Option("inputsJson"), + Option("optionsJson"), + Option(List(Label("labelKey", "labelValue"))), + Option(tempFile) + ), + Map( + "workflowSource" -> "wdl", + "workflowType" -> "wfType", + "workflowTypeVersion" -> "wfTypeVersion", + "workflowInputs" -> "inputsJson", + "workflowOptions" -> "optionsJson", + "labels" -> """{"labelKey":"labelValue"}""" + ), + Map("workflowDependencies" -> tempFile) ), - ("submit a wdl using workflow url", - WorkflowSingleSubmission( - None, - Option("https://link-to-url"), - None, - Option("wfType"), - Option("wfTypeVersion"), - Option("inputsJson"), - Option("optionsJson"), - Option(List(Label("labelKey", "labelValue"))), - Option(tempFile) - ), - Map( - "workflowUrl" -> "https://link-to-url", - "workflowType" -> "wfType", - "workflowTypeVersion" -> "wfTypeVersion", - "workflowInputs" -> "inputsJson", - "workflowOptions" -> "optionsJson", - "labels" -> """{"labelKey":"labelValue"}""" - ), - Map("workflowDependencies" -> tempFile) + WorkflowSingleSubmission( + None, + Option("https://link-to-url"), + None, + Option("wfType"), + Option("wfTypeVersion"), + Option("inputsJson"), + Option("optionsJson"), + Option(List(Label("labelKey", "labelValue"))), + Option(tempFile) + ), + Map( + "workflowUrl" -> "https://link-to-url", + "workflowType" -> "wfType", + "workflowTypeVersion" -> "wfTypeVersion", + "workflowInputs" -> "inputsJson", + "workflowOptions" -> "optionsJson", + "labels" -> """{"labelKey":"labelValue"}""" + ), + Map("workflowDependencies" -> tempFile) ), - ("batch submit a wdl with data", - WorkflowBatchSubmission( - Option("wdl"), - None, - None, - Option("wfType"), - Option("wfTypeVersion"), - List("inputsJson1", "inputsJson2"), - Option("optionsJson"), - Option(List(Label("labelKey", "labelValue"))), - Option(tempFile) - ), - Map( - "workflowSource" -> "wdl", - "workflowType" -> "wfType", - "workflowTypeVersion" -> "wfTypeVersion", - "workflowInputs" -> "[inputsJson1,inputsJson2]", - "workflowOptions" -> "optionsJson", - "labels" -> """{"labelKey":"labelValue"}""" - ), - Map("workflowDependencies" -> tempFile) + WorkflowBatchSubmission( + Option("wdl"), + None, + None, + Option("wfType"), + Option("wfTypeVersion"), + List("inputsJson1", "inputsJson2"), + Option("optionsJson"), + Option(List(Label("labelKey", "labelValue"))), + Option(tempFile) + ), + Map( + "workflowSource" -> "wdl", + "workflowType" -> "wfType", + "workflowTypeVersion" -> "wfTypeVersion", + "workflowInputs" -> "[inputsJson1,inputsJson2]", + "workflowOptions" -> "optionsJson", + "labels" -> """{"labelKey":"labelValue"}""" + ), + Map("workflowDependencies" -> tempFile) ) ) @@ -132,24 +126,22 @@ class CromwellClientSpec extends AsyncFlatSpec with BeforeAndAfterAll with Match contentType.mediaType.isMultipart should be(true) val boundary = contentType.mediaType.params("boundary") - val expectedJsonChunks = expectedJsons map { - case (chunkKey, chunkValue) => - s"""|--$boundary - |Content-Type: application/json - |Content-Disposition: form-data; name="$chunkKey" - | - |$chunkValue - |""".stripMargin.replace("\n", "\r\n").trim + val expectedJsonChunks = expectedJsons map { case (chunkKey, chunkValue) => + s"""|--$boundary + |Content-Type: application/json + |Content-Disposition: form-data; name="$chunkKey" + | + |$chunkValue + |""".stripMargin.replace("\n", "\r\n").trim } - val expectedFileChunks = expectedFiles.iterator map { - case (chunkKey, chunkFile) => - s"""|--$boundary - |Content-Type: application/zip - |Content-Disposition: form-data; filename="${chunkFile.name}"; name="$chunkKey" - |""".stripMargin.replace("\n", "\r\n").trim + val expectedFileChunks = expectedFiles.iterator map { case (chunkKey, chunkFile) => + s"""|--$boundary + |Content-Type: application/zip + |Content-Disposition: form-data; filename="${chunkFile.name}"; name="$chunkKey" + |""".stripMargin.replace("\n", "\r\n").trim } - val expectedFileContents = expectedFiles.iterator map { - case (_, chunkFile) => chunkFile.contentAsString + val expectedFileContents = expectedFiles.iterator map { case (_, chunkFile) => + chunkFile.contentAsString } val boundaryEnd = s"--$boundary--" diff --git a/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala index 06841bfad71..34e901857b7 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/CromwellResponseFailedSpec.scala @@ -15,15 +15,18 @@ import org.scalatest.matchers.should.Matchers import scala.concurrent.duration._ import scala.concurrent.{Await, Future} -class CromwellResponseFailedSpec extends TestKit(ActorSystem("CromwellResponseFailedSpec")) - with AsyncFlatSpecLike with Matchers with BeforeAndAfterAll { +class CromwellResponseFailedSpec + extends TestKit(ActorSystem("CromwellResponseFailedSpec")) + with AsyncFlatSpecLike + with Matchers + with BeforeAndAfterAll { override def afterAll(): Unit = { Await.ready(system.terminate(), 10.seconds.dilated) super.afterAll() } - + implicit val materializer: ActorMaterializer = ActorMaterializer() - + "CromwellAPIClient" should "fail the Future if the HttpResponse is unsuccessful" in { val errorMessage = """|{ @@ -32,14 +35,15 @@ class CromwellResponseFailedSpec extends TestKit(ActorSystem("CromwellResponseFa |} |""".stripMargin.trim val client = new CromwellClient(new URL("http://fakeurl"), "v1") { - override def executeRequest(request: HttpRequest, headers: List[HttpHeader]): Future[HttpResponse] = Future.successful( - new HttpResponse( - StatusCodes.ServiceUnavailable, - List.empty[HttpHeader], - HttpEntity(ContentTypes.`application/json`, errorMessage), - HttpProtocols.`HTTP/1.1` + override def executeRequest(request: HttpRequest, headers: List[HttpHeader]): Future[HttpResponse] = + Future.successful( + new HttpResponse( + StatusCodes.ServiceUnavailable, + List.empty[HttpHeader], + HttpEntity(ContentTypes.`application/json`, errorMessage), + HttpProtocols.`HTTP/1.1` + ) ) - ) } for { diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala index 9ac224970bf..4eb0a450aca 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/model/CromwellQueryResultJsonFormatterSpec.scala @@ -7,50 +7,51 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class CromwellQueryResultJsonFormatterSpec extends AnyFlatSpec with Matchers { behavior of "CromwellQueryResultJsonFormat" - val sampleQueryResult = CromwellQueryResults(results = List( - CromwellQueryResult( - Option("switcheroo"), - WorkflowId.fromString("bee51f36-396d-4e22-8a81-33dedff66bf6"), - Failed, - Option(OffsetDateTime.parse("2017-07-24T14:44:34.010Z")), + val sampleQueryResult = CromwellQueryResults(results = + List( + CromwellQueryResult( + Option("switcheroo"), + WorkflowId.fromString("bee51f36-396d-4e22-8a81-33dedff66bf6"), + Failed, + Option(OffsetDateTime.parse("2017-07-24T14:44:34.010Z")), Option(OffsetDateTime.parse("2017-07-24T14:44:33.227Z")), - "Archived" - ), - CromwellQueryResult( - Option("switcheroo"), - WorkflowId.fromString("0071495e-39eb-478e-bc98-8614b986c91e"), - Succeeded, + "Archived" + ), + CromwellQueryResult( + Option("switcheroo"), + WorkflowId.fromString("0071495e-39eb-478e-bc98-8614b986c91e"), + Succeeded, Option(OffsetDateTime.parse("2017-07-24T15:06:45.940Z")), - Option(OffsetDateTime.parse("2017-07-24T15:04:54.372Z")), - "Unarchived" - ), - )) - - val sampleJson = """|{ - | "results": [ - | { - | "name": "switcheroo", - | "id": "bee51f36-396d-4e22-8a81-33dedff66bf6", - | "status": "Failed", - | "end": "2017-07-24T14:44:34.010Z", - | "start": "2017-07-24T14:44:33.227Z", - | "metadataArchiveStatus": "Archived" - | }, - | { - | "name": "switcheroo", - | "id": "0071495e-39eb-478e-bc98-8614b986c91e", - | "status": "Succeeded", - | "end": "2017-07-24T15:06:45.940Z", - | "start": "2017-07-24T15:04:54.372Z", - | "metadataArchiveStatus": "Unarchived" - | } - | ] - |}""".stripMargin.parseJson.asJsObject + Option(OffsetDateTime.parse("2017-07-24T15:04:54.372Z")), + "Unarchived" + ) + ) + ) + + val sampleJson = """|{ + | "results": [ + | { + | "name": "switcheroo", + | "id": "bee51f36-396d-4e22-8a81-33dedff66bf6", + | "status": "Failed", + | "end": "2017-07-24T14:44:34.010Z", + | "start": "2017-07-24T14:44:33.227Z", + | "metadataArchiveStatus": "Archived" + | }, + | { + | "name": "switcheroo", + | "id": "0071495e-39eb-478e-bc98-8614b986c91e", + | "status": "Succeeded", + | "end": "2017-07-24T15:06:45.940Z", + | "start": "2017-07-24T15:04:54.372Z", + | "metadataArchiveStatus": "Unarchived" + | } + | ] + |}""".stripMargin.parseJson.asJsObject it should "write a query result as a structured JsObject" in { sampleQueryResult.toJson shouldEqual sampleJson diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala index c1f1e89c08a..ab0c7e08cbc 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/model/LabelsJsonFormatterSpec.scala @@ -4,18 +4,17 @@ import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import spray.json._ - class LabelsJsonFormatterSpec extends AnyFlatSpec with Matchers { import cromwell.api.model.LabelsJsonFormatter._ behavior of "WdlValueJsonFormat" val sampleLabels = List(Label("key-1", "value-1"), Label("key-2", "value-2"), Label("key-3", "value-3")) - val sampleJson = """|{ - | "key-1":"value-1", - | "key-2":"value-2", - | "key-3":"value-3" - |}""".stripMargin.parseJson.asJsObject + val sampleJson = """|{ + | "key-1":"value-1", + | "key-2":"value-2", + | "key-3":"value-3" + |}""".stripMargin.parseJson.asJsObject it should "write a Label as a structured JsObject" in { val label = List(Label("test-key", "test-value")) diff --git a/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala b/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala index 5bad5f8b21f..7e328ee4725 100644 --- a/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala +++ b/cromwellApiClient/src/test/scala/cromwell/api/model/WaasDescriptionJsonSupportSpec.scala @@ -3,7 +3,6 @@ package cromwell.api.model import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers - class WaasDescriptionJsonSupportSpec extends AnyFlatSpec with Matchers { it should "deserialize invalid result JSON" in { @@ -25,7 +24,6 @@ class WaasDescriptionJsonSupportSpec extends AnyFlatSpec with Matchers { | "isRunnableWorkflow": false |}""".stripMargin - import cromwell.api.model.WorkflowDescriptionJsonSupport._ import spray.json._ @@ -33,7 +31,11 @@ class WaasDescriptionJsonSupportSpec extends AnyFlatSpec with Matchers { val deserialized = jsonAst.convertTo[WaasDescription] deserialized.valid should be(false) - deserialized.errors should be(List("""Failed to import workflow sub_workflow_aborted_import.wdl.:\nBad import sub_workflow_aborted_import.wdl: Failed to resolve 'sub_workflow_aborted_import.wdl' using resolver: 'http importer (no 'relative-to' origin)' (reason 1 of 1): Relative path""")) + deserialized.errors should be( + List( + """Failed to import workflow sub_workflow_aborted_import.wdl.:\nBad import sub_workflow_aborted_import.wdl: Failed to resolve 'sub_workflow_aborted_import.wdl' using resolver: 'http importer (no 'relative-to' origin)' (reason 1 of 1): Relative path""" + ) + ) deserialized.validWorkflow should be(false) deserialized.name should be("") deserialized.inputs should be(List.empty) diff --git a/cwl/src/main/resources/reference.conf b/cwl/src/main/resources/reference.conf deleted file mode 100644 index 0d0d8a980f9..00000000000 --- a/cwl/src/main/resources/reference.conf +++ /dev/null @@ -1,3 +0,0 @@ -cwltool-runner { - class = "cwl.CwltoolHeterodon" -} diff --git a/cwl/src/main/scala/cwl/ArgumentToCommandPart.scala b/cwl/src/main/scala/cwl/ArgumentToCommandPart.scala deleted file mode 100644 index ca67eaa7ab3..00000000000 --- a/cwl/src/main/scala/cwl/ArgumentToCommandPart.scala +++ /dev/null @@ -1,26 +0,0 @@ -package cwl - -import cwl.command.StringCommandPart -import shapeless._ -import wom.CommandPart - -object ArgumentToCommandPart extends Poly1 { - type MakeCommandPart = (Boolean, ExpressionLib) => CommandPart - implicit def caseStringOrExpression: Case.Aux[StringOrExpression, MakeCommandPart] = at { - _.fold(this) - } - - implicit def caseExpression: Case.Aux[Expression, MakeCommandPart] = at { - CwlExpressionCommandPart.apply - } - - implicit def caseString: Case.Aux[String, MakeCommandPart] = at { - string => - (_, _) => - StringCommandPart(string) - } - - implicit def caseCommandLineBinding: Case.Aux[ArgumentCommandLineBinding, MakeCommandPart] = at { - ArgumentCommandLineBindingCommandPart.apply - } -} diff --git a/cwl/src/main/scala/cwl/BaseCommandToCommandParts.scala b/cwl/src/main/scala/cwl/BaseCommandToCommandParts.scala deleted file mode 100644 index 2f597918082..00000000000 --- a/cwl/src/main/scala/cwl/BaseCommandToCommandParts.scala +++ /dev/null @@ -1,11 +0,0 @@ -package cwl - -import cwl.command.StringCommandPart -import shapeless.Poly1 -import wom.CommandPart - -object BaseCommandToCommandParts extends Poly1 { - implicit def one = at[String] { Seq(_).map(StringCommandPart.apply): Seq[CommandPart] } - - implicit def many = at[Array[String]] { _.toSeq.map(StringCommandPart.apply): Seq[CommandPart] } -} diff --git a/cwl/src/main/scala/cwl/CommandLineTool.scala b/cwl/src/main/scala/cwl/CommandLineTool.scala deleted file mode 100644 index 49a2de8c861..00000000000 --- a/cwl/src/main/scala/cwl/CommandLineTool.scala +++ /dev/null @@ -1,439 +0,0 @@ -package cwl - -import cats.data.{NonEmptyList, OptionT} -import cats.syntax.apply._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import cats.instances.future._ -import common.Checked -import common.validation.ErrorOr._ -import cwl.CommandLineTool._ -import cwl.CwlVersion._ -import cwl.internal.CommandPartSortingAlgorithm -import io.circe.Json -import shapeless.syntax.singleton._ -import shapeless.{:+:, CNil, Coproduct, Poly1, Witness} -import wom.callable.CommandTaskDefinition.{EvaluatedOutputs, OutputFunctionResponse} -import wom.callable.{Callable, CallableTaskDefinition, ContainerizedInputExpression} -import wom.expression.{IoFunctionSet, ValueAsAnExpression, WomExpression} -import wom.graph.GraphNodePort.OutputPort -import wom.types.{WomArrayType, WomIntegerType, WomOptionalType} -import wom.util.YamlUtils -import wom.values.{WomArray, WomEvaluatedCallInputs, WomGlobFile, WomInteger, WomString, WomValue} -import wom.{CommandPart, RuntimeAttributes, RuntimeAttributesKeys} - -import scala.concurrent.{ExecutionContext, Future} -import scala.math.Ordering - -/** - * @param `class` This _should_ always be "CommandLineTool," however the spec does not -er- specify this. - */ -case class CommandLineTool private( - inputs: Array[CommandInputParameter], - outputs: Array[CommandOutputParameter], - `class`: Witness.`"CommandLineTool"`.T, - id: String, - requirements: Option[Array[Requirement]], - hints: Option[Array[Hint]], - label: Option[String], - doc: Option[String], - cwlVersion: Option[CwlVersion], - baseCommand: Option[BaseCommand], - arguments: Option[Array[CommandLineTool.Argument]], - stdin: Option[StringOrExpression], - stderr: Option[StringOrExpression], - stdout: Option[StringOrExpression], - successCodes: Option[Array[Int]], - temporaryFailCodes: Option[Array[Int]], - permanentFailCodes: Option[Array[Int]], - `$namespaces`: Option[Map[String, String]], - `$schemas`: Option[Array[String]] - ) extends Tool { - - def asCwl: Cwl = Coproduct[Cwl](this) - - lazy val baseCommandPart: List[CommandPart] = baseCommand.toList.flatMap(_.fold(BaseCommandToCommandParts)) - - /* - * The command template is built following the rules described here: http://www.commonwl.org/v1.0/CommandLineTool.html#Input_binding - * - The baseCommand goes first - * - Then the arguments are assigned a sorting key and transformed into a CommandPart - * - Finally the inputs are folded one by one into a CommandPartsList - * - arguments and inputs CommandParts are sorted according to their sort key - */ - private [cwl] def buildCommandTemplate: CommandPartExpression[List[CommandPart]] = - ( - CommandPartSortingAlgorithm.inputBindingsCommandParts(inputs), - CommandPartSortingAlgorithm.argumentCommandParts(arguments) - ).mapN ( _ ++ _ ). - //sort the highest level, then recursively pull in the nested (and sorted) command parts within each SortKeyAndCommandPart - map{ _.sorted.flatMap(_.sortedCommandParts) }. - map(baseCommandPart ++ _) - - // This seems like it makes sense only for CommandLineTool and hence is not abstracted in Tool. If this assumption is wrong it could be moved up. - private def environmentDefs(requirementsAndHints: List[Requirement], expressionLib: ExpressionLib): ErrorOr[Map[String, WomExpression]] = { - // For environment variables we need to make sure that we aren't being asked to evaluate expressions from a containing - // workflow step or its containing workflow or anything containing the workflow. The current structure of this code - // is not prepared to evaluate those expressions. Actually this is true for attributes too and we're totally not - // checking for this condition there. Blurgh. - // TODO CWL: for runtime attributes, detect unevaluatable expressions in the containment hierarchy. - - // This traverses all `EnvironmentDef`s within all `EnvVarRequirement`s. The spec doesn't appear to say how to handle - // duplicate `envName` keys in a single array of `EnvironmentDef`s; this code gives precedence to the last occurrence. - val allEnvVarDefs = for { - req <- requirementsAndHints - envVarReq <- req.select[EnvVarRequirement].toList - // Reverse the defs within an env var requirement so that when we fold from the right below the later defs - // will take precedence over the earlier defs. - envDef <- envVarReq.envDef.toList.reverse - } yield envDef - - // Compact the `EnvironmentDef`s. Don't convert to `WomExpression`s yet, the `StringOrExpression`s need to be - // compared to the `EnvVarRequirement`s that were defined on this tool. - val effectiveEnvironmentDefs = allEnvVarDefs.foldRight(Map.empty[String, StringOrExpression]) { - case (envVarReq, envVarMap) => envVarMap + (envVarReq.envName -> envVarReq.envValue) - } - - // These are the effective environment defs irrespective of where they were found in the - // Run / WorkflowStep / Workflow containment hierarchy. - val effectiveExpressionEnvironmentDefs = effectiveEnvironmentDefs filter { case (_, expr) => expr.select[Expression].isDefined } - - // These are only the environment defs defined on this tool. - val cltRequirements = requirements.toList.flatten ++ hints.toList.flatten.flatMap(_.select[Requirement]) - val cltEnvironmentDefExpressions = (for { - cltEnvVarRequirement <- cltRequirements flatMap { - _.select[EnvVarRequirement] - } - cltEnvironmentDef <- cltEnvVarRequirement.envDef.toList - expr <- cltEnvironmentDef.envValue.select[Expression].toList - } yield expr).toSet - - // If there is an expression in an effective environment def that wasn't defined on this tool then error out since - // there isn't currently a way of evaluating it. - val unevaluatableEnvironmentDefs = for { - (name, stringOrExpression) <- effectiveExpressionEnvironmentDefs.toList - expression <- stringOrExpression.select[Expression].toList - if !cltEnvironmentDefExpressions.contains(expression) - } yield name - - unevaluatableEnvironmentDefs match { - case Nil => - // No unevaluatable environment defs => keep on truckin' - effectiveEnvironmentDefs.foldRight(Map.empty[String, WomExpression]) { case ((envName, envValue), acc) => - acc + (envName -> envValue.fold(StringOrExpressionToWomExpression).apply(inputNames, expressionLib)) - }.validNel - case xs => - s"Could not evaluate environment variable expressions defined in the call hierarchy of tool $id: ${xs.mkString(", ")}.".invalidNel - } - } - - /* - * Custom evaluation of the outputs of the command line tool (custom as in bypasses the engine provided default implementation). - * This is needed because the output of a CWL tool might be defined by the presence of a cwl.output.json file in the output directory. - * In that case, the content of the file should be used as output. This means that otherwise provided output expressions should not be evaluated. - * This method checks for the existence of this file, and optionally return a Map[OutputPort, WomValue] if the file was found. - * It ensures all the output ports of the corresponding WomCallNode get a WomValue which is needed for the engine to keep running the workflow properly. - * If the json file happens to be missing values for one or more output ports, it's a failure. - * If the file is not present, an empty value is returned and the engine will execute its own output evaluation logic. - * - * TODO: use IO instead of Future ? - */ - final private def outputEvaluationJsonFunction(outputPorts: Set[OutputPort], - inputs: Map[String, WomValue], - ioFunctionSet: IoFunctionSet, - executionContext: ExecutionContext): OutputFunctionResponse = { - implicit val ec = executionContext - import cats.syntax.either._ - - // Convert the parsed json to Wom-usable output values - def jsonToOutputs(json: Map[String, Json]): Checked[List[(OutputPort, WomValue)]] = { - - outputPorts.toList.traverse[ErrorOr, (OutputPort, WomValue)]({ outputPort => - // If the type is optional, then we can set the value to none if there's nothing in the json - def emptyValue = outputPort.womType match { - case optional: WomOptionalType => Option((outputPort -> optional.none).validNel) - case _ => None - } - - json.get(outputPort.internalName) - .map( - _.foldWith(CwlJsonToDelayedCoercionFunction).apply(outputPort.womType) - .map(outputPort -> _) - ) - .orElse(emptyValue) - .getOrElse(s"Cannot find a value for output ${outputPort.internalName} in output json $json".invalidNel) - }).toEither - } - - // Parse content as json and return output values for each output port - def parseContent(content: String): EvaluatedOutputs = { - val yaml = YamlUtils.parse(content) - for { - parsed <- yaml.flatMap(_.as[Map[String, Json]]).leftMap(error => NonEmptyList.one(error.getMessage)) - jobOutputsMap <- jsonToOutputs(parsed) - } yield jobOutputsMap.toMap - } - - for { - // Glob for "cwl.output.json" - outputJsonGlobs <- OptionT.liftF { ioFunctionSet.glob(CwlOutputJson) } - // There can only be 0 or 1, so try to take the head of the list - outputJsonFile <- OptionT.fromOption[Future] { outputJsonGlobs.headOption } - // Read the content using the ioFunctionSet.readFile function - content <- OptionT.liftF { ioFunctionSet.readFile(outputJsonFile, None, failOnOverflow = false) } - // parse the content and validate it - outputs = parseContent(content) - } yield outputs - } - - override protected def toolAttributes: Map[String, WomExpression] = { - val codes: List[Int] = successCodes match { - case Some(c) => c.toList // Use the provided list of success codes. - case None => List(0) // Default to allowing only 0 for a success code. - } - - val arr = WomArray(WomArrayType(WomIntegerType), codes map WomInteger.apply) - Map(RuntimeAttributesKeys.ContinueOnReturnCodeKey -> ValueAsAnExpression(arr)) - } - - override def buildTaskDefinition(taskName: String, - inputDefinitions: List[_ <: Callable.InputDefinition], - outputDefinitions: List[Callable.OutputDefinition], - runtimeAttributes: RuntimeAttributes, - requirementsAndHints: List[cwl.Requirement], - expressionLib: ExpressionLib): ErrorOr[CallableTaskDefinition] = { - - def redirect(soe: Option[StringOrExpression]): Option[WomExpression] = soe map { - _.fold(StringOrExpressionToWomExpression).apply(inputNames, expressionLib) - } - - // This seems like it makes sense only for CommandLineTool and hence is not abstracted in Tool. If this assumption is wrong it could be moved up. - val adHocFileCreations: Set[ContainerizedInputExpression] = (for { - requirements <- requirements.getOrElse(Array.empty[Requirement]) - initialWorkDirRequirement <- requirements.select[InitialWorkDirRequirement].toArray - listing <- initialWorkDirRequirement.listings - } yield InitialWorkDirFileGeneratorExpression(listing, expressionLib)).toSet[ContainerizedInputExpression] - - val dockerOutputDirectory = requirementsAndHints - .flatMap(_.select[DockerRequirement]) - .flatMap(_.dockerOutputDirectory) - .headOption - - def inputsToCommandParts(inputs: WomEvaluatedCallInputs) : ErrorOr[Seq[CommandPart]] = - buildCommandTemplate.run((RequirementsAndHints(requirementsAndHints), expressionLib, inputs)) - - environmentDefs(requirementsAndHints, expressionLib) map { environmentExpressions => - CallableTaskDefinition( - taskName, - inputsToCommandParts, - runtimeAttributes, - Map.empty, - Map.empty, - outputDefinitions, - inputDefinitions, - adHocFileCreation = adHocFileCreations, - environmentExpressions = environmentExpressions, - // TODO: This doesn't work in all cases and it feels clunky anyway - find a way to sort that out - prefixSeparator = "#", - commandPartSeparator = " ", - stdinRedirection = redirect(stdin), - stdoutOverride = redirect(stdout), - stderrOverride = redirect(stderr), - additionalGlob = Option(WomGlobFile(CwlOutputJson)), - customizedOutputEvaluation = outputEvaluationJsonFunction, - homeOverride = Option(_.outputPath), - dockerOutputDirectory = dockerOutputDirectory, - sourceLocation = None - ) - } - } -} - -object CommandLineTool { - val CwlOutputJson = "cwl.output.json" - - val DefaultPosition = Coproduct[StringOrInt](0) - - // Elements of the sorting key can be either Strings or Ints - type StringOrInt = String :+: Int :+: CNil - object StringOrInt { - object String { def unapply(stringOrInt: StringOrInt): Option[String] = stringOrInt.select[String] } - object Int { def unapply(stringOrInt: StringOrInt): Option[Int] = stringOrInt.select[Int] } - } - - /* - * The algorithm described here http://www.commonwl.org/v1.0/CommandLineTool.html#Input_binding to sort the command line parts - * uses a sorting key assigned to each binding. This class represents such a key - */ - object CommandBindingSortingKey { - def empty = CommandBindingSortingKey(List.empty, List.empty) - } - case class CommandBindingSortingKey(head: List[StringOrInt], - tail: List[StringOrInt] = List.empty) { - val value = head ++ tail - - def append(binding: Option[CommandLineBinding], name: StringOrInt): CommandBindingSortingKey = binding match { - // If there's an input binding, add the position to the key (or 0) - case Some(b) => - // The spec is inconsistent about this as it says "If position is not specified, it is not added to the sorting key" - // but also that the position defaults to 0. cwltool uses 0 when there's no position so we'll do that too. - val position = b.position.map(Coproduct[StringOrInt](_)) getOrElse DefaultPosition - copy(head = head :+ position, tail = tail :+ name) - // Otherwise do nothing - case None => this - } - - /** - * Creates a new key with head and tail combined into the new head. - */ - def asNewKey = CommandBindingSortingKey(value) - } - - // Maps a sorting key to its (sorted) bindings - case class SortKeyAndCommandPart( - sortingKey: CommandBindingSortingKey, - commandPart: CommandPart, - nestedCommandParts: List[SortKeyAndCommandPart] = List.empty) { - def sortedCommandParts:List[CommandPart] = - List(commandPart) ++ nestedCommandParts.sorted.flatMap(_.sortedCommandParts) - } - - type CommandPartsList = List[SortKeyAndCommandPart] - - // Ordering for CommandBindingSortingKeyElement - implicit val SortingKeyTypeOrdering: Ordering[StringOrInt] = Ordering.fromLessThan[StringOrInt]({ - // String comparison - case (StringOrInt.String(s1), StringOrInt.String(s2)) => s1 < s2 - // Int comparison - case (StringOrInt.Int(i1), StringOrInt.Int(i2)) => i1 < i2 - // Int < String (from the spec: "Numeric entries sort before strings.") - case (StringOrInt.Int(_), StringOrInt.String(_)) => true - // String > Int - case (StringOrInt.String(_), StringOrInt.Int(_)) => false - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - }) - - // https://github.com/scala/bug/issues/4097#issuecomment-292388627 - implicit def IterableSubclass[CC[X] <: Iterable[X], T: Ordering] : Ordering[CC[T]] = { - new Ordering[CC[T]] { - val ord = implicitly[Ordering[T]] - def compare(x: CC[T], y: CC[T]): Int = { - val xe = x.iterator - val ye = y.iterator - - while (xe.hasNext && ye.hasNext) { - val res = ord.compare(xe.next(), ye.next()) - if (res != 0) return res - } - - Ordering.Boolean.compare(xe.hasNext, ye.hasNext) - } - } - } - - // Ordering for a CommandBindingSortingKey - implicit val SortingKeyOrdering: Ordering[CommandBindingSortingKey] = Ordering.by(_.value) - - // Ordering for a CommandPartSortMapping: order by sorting key - implicit val SortKeyAndCommandPartOrdering: Ordering[SortKeyAndCommandPart] = Ordering.by(_.sortingKey) - - def apply(inputs: Array[CommandInputParameter] = Array.empty, - outputs: Array[CommandOutputParameter] = Array.empty, - id: String, - requirements: Option[Array[Requirement]] = None, - hints: Option[Array[Hint]] = None, - label: Option[String] = None, - doc: Option[String] = None, - cwlVersion: Option[CwlVersion] = Option(CwlVersion.Version1), - baseCommand: Option[BaseCommand] = None, - arguments: Option[Array[CommandLineTool.Argument]] = None, - stdin: Option[StringOrExpression] = None, - stderr: Option[StringOrExpression] = None, - stdout: Option[StringOrExpression] = None, - successCodes: Option[Array[Int]] = None, - temporaryFailCodes: Option[Array[Int]] = None, - permanentFailCodes: Option[Array[Int]] = None, - namespaces: Option[Map[String, String]] = None, - schemas: Option[Array[String]] = None - ): CommandLineTool = { - CommandLineTool( - inputs, - outputs, - "CommandLineTool".narrow, - id, - requirements, - hints, - label, - doc, - cwlVersion, - baseCommand, - arguments, - stdin, - stderr, - stdout, - successCodes, - temporaryFailCodes, - permanentFailCodes, - namespaces, - schemas - ) - } - - type BaseCommand = SingleOrArrayOfStrings - - type Argument = ArgumentCommandLineBinding :+: StringOrExpression :+: CNil - - case class CommandInputParameter( - id: String, - label: Option[String] = None, - secondaryFiles: Option[SecondaryFiles] = None, - format: Option[InputParameterFormat] = None, //only valid when type: File - streamable: Option[Boolean] = None, //only valid when type: File - doc: Option[Doc] = None, - inputBinding: Option[InputCommandLineBinding] = None, - default: Option[CwlAny] = None, - `type`: Option[MyriadInputType] = None) extends InputParameter - - case class CommandOutputParameter( - id: String, - label: Option[String] = None, - secondaryFiles: Option[SecondaryFiles] = None, - format: Option[OutputParameterFormat] = None, //only valid when type: File - streamable: Option[Boolean] = None, //only valid when type: File - doc: Option[Doc] = None, - outputBinding: Option[CommandOutputBinding] = None, - `type`: Option[MyriadOutputType] = None) extends OutputParameter { - - /** Overridden to strip off the intentionally uniquified bits and leave only the stuff that we want to look - * at for the purposes of determining cache hits. - * - * before: - * - * CommandOutputParameter(file:///var/folders/qh/vvrlr2q92mvb9bb45sttxll4y3gg2g/T/e17762d1-9921-46c5-833e-cb47e3c3bdfd.temp.1197611185902619026/e17762d1-9921-46c5-833e-cb47e3c3bdfd.cwl#ps/ea5165fc-8948-43ae-8dec-3c0468b56bcb/ps-stdOut,None,None,None,None,None,Some(CommandOutputBinding(Some(Inl(Inr(Inl(ps-stdOut.txt)))),None,None)),Some(Inl(Inl(File)))) - * - * after: - * - * CommandOutputParameter(ps-stdOut,None,None,None,None,None,Some(CommandOutputBinding(Some(Inl(Inr(Inl(ps-stdOut.txt)))),None,None)),Some(Inl(Inl(File)))) - * - * This is possibly too strict (i.e. some of these fields may be irrelevant for cache hit determination), but preferable - * to having false positives. - * Also two coproduct types that can be either single or Arrays have custom stringifying folds for arrays. */ - override def cacheString: String = { - val cacheableId: String = id.substring(id.lastIndexOf('/') + 1) - val cacheableSecondaryFiles = secondaryFiles map { _.fold(SecondaryFilesCacheableString)} - val cacheableType: Option[String] = `type`.map(_.fold(MyriadOutputTypeCacheableString)) - s"CommandOutputParameter($cacheableId,$label,$cacheableSecondaryFiles,$format,$streamable,$doc,$outputBinding,$cacheableType)" - } - } -} - -object StringOrExpressionToWomExpression extends Poly1 { - implicit def string: Case.Aux[String, (Set[String], ExpressionLib) => WomExpression] = at[String] { s => (inputNames, expressionLib) => - ValueAsAnExpression(WomString(s)) - } - - implicit def expression: Case.Aux[Expression, (Set[String], ExpressionLib) => WomExpression] = at[Expression] { e => (inputNames, expressionLib) => - cwl.ECMAScriptWomExpression(e, inputNames, expressionLib) - } -} diff --git a/cwl/src/main/scala/cwl/CommandOutputBinding.scala b/cwl/src/main/scala/cwl/CommandOutputBinding.scala deleted file mode 100644 index bab616f2995..00000000000 --- a/cwl/src/main/scala/cwl/CommandOutputBinding.scala +++ /dev/null @@ -1,295 +0,0 @@ -package cwl - -import cats.effect.IO -import cats.effect.IO._ -import cats.syntax.functor._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.syntax.parallel._ -import cats.instances.list._ -import common.validation.ErrorOr._ -import common.validation.IOChecked._ -import common.validation.Validation._ -import wom.expression.IoFunctionSet.{IoDirectory, IoFile} -import wom.expression.{FileEvaluation, IoFunctionSet} -import wom.types._ -import wom.values._ - -/** @see CommandOutputBinding */ -case class CommandOutputBinding(glob: Option[Glob] = None, - loadContents: Option[Boolean] = None, - outputEval: Option[StringOrExpression] = None) - -object CommandOutputBinding { - /** - * TODO: WOM: WOMFILE: Need to support returning globs for primary and secondary. - * - * Right now, when numerous glob files are created, the backends place each glob result into a different subdirectory. - * Later, when the secondary files are being generated each of their globs are in different sub-directories. - * Unfortunately, the secondary file resolution assumes that the secondaries are _next to_ the primaries, not in - * sibling directories. - * - * While this is being worked on, instead of looking up globs return regular files until this can be fixed. - */ - def isRegularFile(path: String): Boolean = { - path match { - case _ if path.contains("*") => false - case _ if path.contains("?") => false - case _ => true - } - } - - /** - * Returns all the primary and secondary files that _will be_ created by this command output binding. - */ - def getOutputWomFiles(inputValues: Map[String, WomValue], - outputWomType: WomType, - commandOutputBinding: CommandOutputBinding, - secondaryFilesOption: Option[SecondaryFiles], - ioFunctionSet: IoFunctionSet, - expressionLib: ExpressionLib): IOChecked[Set[FileEvaluation]] = { - val parameterContext = ParameterContext(ioFunctionSet, expressionLib, inputs = inputValues) - - /* - CWL can output two types of "glob" path types: - - File will be a glob path - - Directory is a path that will be searched for files. It's assumed that it will not be a glob, but the spec is - unclear and doesn't specifically say how Directories are listed. If the backend uses a POSIX - `find

-type f`, then a globbed path should still work. - - Either way create one of the two types as a return type that the backend should be looking for when the command - completes. - - UPDATE: - It turns out that secondary files can be directories. TODO: WOM: WOMFILE: Not sure what to do with that yet. - https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/search.cwl#L42 - https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/index.py#L36-L37 - */ - def primaryPathsToWomFiles(primaryPaths: List[String]): ErrorOr[List[WomFile]] = { - validate { - primaryPaths map { primaryPath => - val outputWomFlatType = outputWomType match { - case WomMaybeListedDirectoryType => WomUnlistedDirectoryType - case WomArrayType(WomMaybeListedDirectoryType) => WomUnlistedDirectoryType - case _ if isRegularFile(primaryPath) => WomSingleFileType - case _ => WomGlobFileType - } - - WomFile(outputWomFlatType, primaryPath) - } - } - } - - def secondaryFilesToWomFiles(primaryWomFiles: List[WomFile], ioFunctionSet: IoFunctionSet): IOChecked[List[WomFile]] = { - primaryWomFiles.flatTraverse[IOChecked, WomFile] { primaryWomFile => - FileParameter.secondaryFiles(primaryWomFile, - primaryWomFile.womFileType, secondaryFilesOption, parameterContext, expressionLib, ioFunctionSet) - } - } - - for { - primaryPaths <- GlobEvaluator.globs(commandOutputBinding.glob, parameterContext, expressionLib).toIOChecked - primaryWomFiles <- primaryPathsToWomFiles(primaryPaths).toIOChecked - // This sets optional = false arbitrarily for now as this code doesn't have the context to make that determination, - // the caller can change this if necessary. - primaryEvaluations = primaryWomFiles map { FileEvaluation(_, optional = false, secondary = false) } - secondaryWomFiles <- secondaryFilesToWomFiles(primaryWomFiles, ioFunctionSet) - secondaryEvaluations = secondaryWomFiles map { FileEvaluation(_, optional = false, secondary = true) } - } yield (primaryEvaluations ++ secondaryEvaluations).toSet - } - - /** - * Generates an output wom value based on the specification in command output binding. - * - * Depending on the outputWomType, the following steps will be applied as specified in the CWL spec: - * 1. glob: get a list the globbed files as our primary files - * 2. loadContents: load the contents of the primary files - * 3. outputEval: pass in the primary files to an expression to generate our return value - * 4. secondaryFiles: just before returning the value, fill in the secondary files on the return value - * - * The result type will be coerced to the output type. - */ - def generateOutputWomValue(inputValues: Map[String, WomValue], - ioFunctionSet: IoFunctionSet, - outputWomType: WomType, - commandOutputBinding: CommandOutputBinding, - secondaryFilesCoproduct: Option[SecondaryFiles], - formatCoproduct: Option[StringOrExpression], - expressionLib: ExpressionLib): IOChecked[WomValue] = { - import ioFunctionSet.cs - - val parameterContext = ParameterContext(ioFunctionSet, expressionLib, inputs = inputValues) - - // 3. outputEval: pass in the primary files to an expression to generate our return value - def evaluateWomValue(womFilesArray: WomArray): ErrorOr[WomValue] = { - commandOutputBinding.outputEval match { - case Some(StringOrExpression.String(string)) => WomString(string).valid - case Some(StringOrExpression.Expression(expression)) => - val outputEvalParameterContext = parameterContext.copy(self = womFilesArray) - ExpressionEvaluator.eval(expression, outputEvalParameterContext) - case None => - womFilesArray.valid - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - } - - // Used to retrieve the file format to be injected into a file result. - def formatOptionErrorOr = OutputParameter.format(formatCoproduct, parameterContext, expressionLib) - - // 4. secondaryFiles: just before returning the value, fill in the secondary files on the return value - def populateSecondaryFiles(evaluatedWomValue: WomValue): IOChecked[WomValue] = { - for { - formatOption <- formatOptionErrorOr.toIOChecked - womValue <- FileParameter.populateSecondaryFiles( - evaluatedWomValue, - secondaryFilesCoproduct, - formatOption, - parameterContext, - expressionLib, - ioFunctionSet - ) - } yield womValue - } - - // CWL tells us the type this output is expected to be. Attempt to coerce the actual output into this type. - def coerceWomValue(populatedWomValue: WomValue): ErrorOr[WomValue] = { - (outputWomType, populatedWomValue) match { - - case (womType: WomArrayType, womValue: WomArray) => - // Array -from- Array then coerce normally - // NOTE: this evaluates to the same as the last case, but guards against the next cases accidentally matching - womType.coerceRawValue(womValue).toErrorOr - - case (womType: WomArrayType, womValue) => - // Array -from- Single then coerce a single value to an array - womType.coerceRawValue(womValue).toErrorOr.map(value => WomArray(womType, List(value))) - - case (womType, womValue: WomArray) if womValue.value.lengthCompare(1) == 0 => - // Single -from- Array then coerce the head (if there's only a head) - womType.coerceRawValue(womValue.value.head).toErrorOr - - case (womType, womValue) => - // to then coerce normally - womType.coerceRawValue(womValue).toErrorOr - - } - } - - for { - // 1. glob: get a list the globbed files as our primary files - primaryPaths <- GlobEvaluator.globs(commandOutputBinding.glob, parameterContext, expressionLib).toIOChecked - - // 2. loadContents: load the contents of the primary files - primaryAsDirectoryOrFiles <- primaryPaths.parTraverse[IOChecked, List[WomFile]] { - loadPrimaryWithContents(ioFunctionSet, outputWomType, commandOutputBinding) - } map (_.flatten) - - // Make globbed files absolute paths by prefixing them with the output dir if necessary - absolutePaths = primaryAsDirectoryOrFiles.map(_.mapFile(ioFunctionSet.pathFunctions.relativeToHostCallRoot)) - - // Load file size - withFileSizes <- absolutePaths.parTraverse[IOChecked, WomFile](_.withSize(ioFunctionSet).to[IOChecked]) - - womFilesArray = WomArray(withFileSizes) - - // 3. outputEval: pass in the primary files to an expression to generate our return value - evaluatedWomValue <- evaluateWomValue(womFilesArray).toIOChecked - - // 4. secondaryFiles: just before returning the value, fill in the secondary files on the return value - populatedWomValue <- populateSecondaryFiles(evaluatedWomValue) - - // CWL tells us the type this output is expected to be. Attempt to coerce the actual output into this type. - coercedWomValue <- coerceWomValue(populatedWomValue).toIOChecked - } yield coercedWomValue - } - - /** - * Given a cwl glob path and an output type, gets the listing using the ioFunctionSet, and optionally loads the - * contents of the file(s). - */ - private def loadPrimaryWithContents(ioFunctionSet: IoFunctionSet, - outputWomType: WomType, - commandOutputBinding: CommandOutputBinding) - (cwlPath: String): IOChecked[List[WomFile]] = { - import ioFunctionSet.cs - - /* - For each file matched in glob, read up to the first 64 KiB of text from the file and place it in the contents field - of the file object for manipulation by outputEval. - */ - val womMaybeListedDirectoryOrFileType = outputWomType match { - case WomMaybeListedDirectoryType => WomMaybeListedDirectoryType - case WomArrayType(WomMaybeListedDirectoryType) => WomMaybeListedDirectoryType - case _ => WomMaybePopulatedFileType - } - womMaybeListedDirectoryOrFileType match { - case WomMaybeListedDirectoryType => - // Even if multiple directories are _somehow_ requested, a single flattened directory is returned. - loadDirectoryWithListing(ioFunctionSet, commandOutputBinding)(cwlPath).map(List(_)) - case WomMaybePopulatedFileType if isRegularFile(cwlPath) => - loadFileWithContents(ioFunctionSet, commandOutputBinding)(cwlPath).to[IOChecked].map(List(_)) - case WomMaybePopulatedFileType => - //TODO: HACK ALERT - DB: I am starting on ticket https://github.com/broadinstitute/cromwell/issues/3092 which will redeem me of this mortal sin. - val detritusFiles = List( - "docker_cid", - "gcs_delocalization.sh", - "gcs_localization.sh", - "gcs_transfer.sh", - "rc.tmp", - "script", - "script.background", - "script.submit", - "stderr", - "stderr.background", - "stdout", - "stdout.background", - ) - val globs: IOChecked[Seq[String]] = - ioFunctionSet.glob(cwlPath).toIOChecked - .map({ - _ - .filterNot{ s => - detritusFiles exists s.endsWith - } - }) - - globs.flatMap({ files => - files.toList.parTraverse[IOChecked, WomFile](v => loadFileWithContents(ioFunctionSet, commandOutputBinding)(v).to[IOChecked]) - }) - case other => s"Program error: $other type was not expected".invalidIOChecked - } - } - - /** - * Loads a directory with the files listed, each file with contents populated. - */ - private def loadDirectoryWithListing(ioFunctionSet: IoFunctionSet, - commandOutputBinding: CommandOutputBinding)(path: String, visited: Vector[String] = Vector.empty): IOChecked[WomMaybeListedDirectory] = { - import ioFunctionSet.cs - - for { - listing <- IO.fromFuture(IO { ioFunctionSet.listDirectory(path)(visited) }).to[IOChecked] - loadedListing <- listing.toList.parTraverse[IOChecked, WomFile]({ - case IoFile(p) => loadFileWithContents(ioFunctionSet, commandOutputBinding)(p).to[IOChecked] - case IoDirectory(p) => loadDirectoryWithListing(ioFunctionSet, commandOutputBinding)(p, visited :+ path).widen - }) - } yield WomMaybeListedDirectory(valueOption = Option(path), listingOption = Option(loadedListing)) - } - - /** - * Loads a file at path reading 64KiB of data into the contents. - */ - private def loadFileWithContents(ioFunctionSet: IoFunctionSet, - commandOutputBinding: CommandOutputBinding)(path: String): IO[WomFile] = { - val contentsOptionErrorOr = { - if (commandOutputBinding.loadContents getOrElse false) { - FileParameter.load64KiB(path, ioFunctionSet) map Option.apply - } else { - IO.pure(None) - } - } - contentsOptionErrorOr map { contentsOption => - WomMaybePopulatedFile(valueOption = Option(path), contentsOption = contentsOption) - } - } -} diff --git a/cwl/src/main/scala/cwl/CwlCodecs.scala b/cwl/src/main/scala/cwl/CwlCodecs.scala deleted file mode 100644 index d7010494bca..00000000000 --- a/cwl/src/main/scala/cwl/CwlCodecs.scala +++ /dev/null @@ -1,153 +0,0 @@ -package cwl - -import cats.data.NonEmptyList -import cats.syntax.either._ -import cats.syntax.show._ -import common.Checked -import common.validation.Checked._ -import cwl.CommandLineTool.{CommandInputParameter, CommandOutputParameter} -import cwl.CwlType.CwlType -import cwl.CwlVersion.CwlVersion -import cwl.ExpressionTool.{ExpressionToolInputParameter, ExpressionToolOutputParameter} -import cwl.InitialWorkDirRequirement.IwdrListingArrayEntry -import cwl.LinkMergeMethod.LinkMergeMethod -import cwl.ScatterMethod.ScatterMethod -import cwl.SchemaDefRequirement.SchemaDefTypes -import cwl.Workflow.{WorkflowInputParameter, WorkflowOutputParameter} -import io.circe.DecodingFailure._ -import io.circe.Json._ -import io.circe._ -import io.circe.generic.semiauto._ -import io.circe.literal._ -import io.circe.refined._ -import io.circe.shapes._ -import shapeless.Coproduct - -object CwlCodecs { - /* - Semi-auto codecs for the Cwl types - https://circe.github.io/circe/codecs/semiauto-derivation.html - - Some types, such as BaseCommand are not listed, as they are actually one-for-one type aliases. Having two exact - implicits for the same type will confuse the compiler, leading to cryptic error messages elsewhere. - - For example, a duplicated decoder such as: - - ``` - implicit lazy val decodeSingleOrArrayOfStrings: Decoder[SingleOrArrayOfStrings] = decodeCCons - implicit lazy val decodeBaseCommand: Decoder[BaseCommand] = decodeCCons - ``` - - causes cryptic errors such as: - - ``` - [error] cwl/src/main/scala/cwl/CwlCodecs.scala:123:456: could not find Lazy implicit value of type io.circe.generic.decoding.DerivedDecoder[A] - [error] implicit lazy val decodeCommandInputParameter: Decoder[CommandInputParameter] = deriveDecoder - - (etc. etc. etc.) - ``` - */ - - // Semi-Automatically derived codecs - implicit lazy val codecArgumentCommandLineBinding: Codec[ArgumentCommandLineBinding] = deriveCodec - implicit lazy val codecCommandInputParameter: Codec[CommandInputParameter] = deriveCodec - implicit lazy val codecCommandLineTool: Codec[CommandLineTool] = deriveCodec - implicit lazy val codecCommandOutputBinding: Codec[CommandOutputBinding] = deriveCodec - implicit lazy val codecCommandOutputParameter: Codec[CommandOutputParameter] = deriveCodec - implicit lazy val codecCwlType: Codec[CwlType] = Codec.codecForEnumeration(CwlType) - implicit lazy val codecCwlVersion: Codec[CwlVersion] = Codec.codecForEnumeration(CwlVersion) - implicit lazy val codecDirectory: Codec[Directory] = deriveCodec - implicit lazy val codecDockerRequirement: Codec[DockerRequirement] = deriveCodec - implicit lazy val codecEnvVarRequirement: Codec[EnvVarRequirement] = deriveCodec - implicit lazy val codecEnvironmentDef: Codec[EnvironmentDef] = deriveCodec - implicit lazy val codecExpressionDirent: Codec[ExpressionDirent] = deriveCodec - implicit lazy val codecExpressionTool: Codec[ExpressionTool] = deriveCodec - implicit lazy val codecExpressionToolInputParameter: Codec[ExpressionToolInputParameter] = deriveCodec - implicit lazy val codecExpressionToolOutputParameter: Codec[ExpressionToolOutputParameter] = deriveCodec - implicit lazy val codecFile: Codec[File] = deriveCodec - implicit lazy val codecInitialWorkDirRequirement: Codec[InitialWorkDirRequirement] = deriveCodec - implicit lazy val codecInlineJavascriptRequirement: Codec[InlineJavascriptRequirement] = deriveCodec - implicit lazy val codecInputArraySchema: Codec[InputArraySchema] = deriveCodec - implicit lazy val codecInputBinding: Codec[InputBinding] = deriveCodec - implicit lazy val codecInputCommandLineBinding: Codec[InputCommandLineBinding] = deriveCodec - implicit lazy val codecInputEnumSchema: Codec[InputEnumSchema] = deriveCodec - implicit lazy val codecInputRecordField: Codec[InputRecordField] = deriveCodec - implicit lazy val codecInputRecordSchema: Codec[InputRecordSchema] = deriveCodec - implicit lazy val codecInputResourceRequirement: Codec[DnaNexusInputResourceRequirement] = deriveCodec - implicit lazy val codecLinkMergeMethod: Codec[LinkMergeMethod] = Codec.codecForEnumeration(LinkMergeMethod) - implicit lazy val codecMultipleInputFeatureRequirement: Codec[MultipleInputFeatureRequirement] = deriveCodec - implicit lazy val codecOutputArraySchema: Codec[OutputArraySchema] = deriveCodec - implicit lazy val codecOutputEnumSchema: Codec[OutputEnumSchema] = deriveCodec - implicit lazy val codecOutputRecordField: Codec[OutputRecordField] = deriveCodec - implicit lazy val codecOutputRecordSchema: Codec[OutputRecordSchema] = deriveCodec - implicit lazy val codecResourceRequirement: Codec[ResourceRequirement] = deriveCodec - implicit lazy val codecScatterFeatureRequirement: Codec[ScatterFeatureRequirement] = deriveCodec - implicit lazy val codecScatterMethod: Codec[ScatterMethod] = Codec.codecForEnumeration(ScatterMethod) - implicit lazy val codecSchemaDefRequirement: Codec[SchemaDefRequirement] = deriveCodec - implicit lazy val codecShellCommandRequirement: Codec[ShellCommandRequirement] = deriveCodec - implicit lazy val codecSoftwarePackage: Codec[SoftwarePackage] = deriveCodec - implicit lazy val codecSoftwareRequirement: Codec[SoftwareRequirement] = deriveCodec - implicit lazy val codecStepInputExpressionRequirement: Codec[StepInputExpressionRequirement] = deriveCodec - implicit lazy val codecStringDirent: Codec[StringDirent] = deriveCodec - implicit lazy val codecSubworkflowFeatureRequirement: Codec[SubworkflowFeatureRequirement] = deriveCodec - implicit lazy val codecWorkflow: Codec[Workflow] = deriveCodec - implicit lazy val codecWorkflowInputParameter: Codec[WorkflowInputParameter] = deriveCodec - implicit lazy val codecWorkflowOutputParameter: Codec[WorkflowOutputParameter] = deriveCodec - implicit lazy val codecWorkflowStep: Codec[WorkflowStep] = deriveCodec - implicit lazy val codecWorkflowStepInput: Codec[WorkflowStepInput] = deriveCodec - implicit lazy val codecWorkflowStepOutput: Codec[WorkflowStepOutput] = deriveCodec - - // Encoders and decoders for Coproduct-based types must be explicitly derived - implicit lazy val decodeCwlAny: Decoder[CwlAny] = decodeCCons - implicit lazy val decodeExpression: Decoder[Expression] = decodeCCons - implicit lazy val decodeFileOrDirectory: Decoder[FileOrDirectory] = decodeCCons - implicit lazy val decodeGlob: Decoder[Glob] = decodeCCons - implicit lazy val decodeHint: Decoder[Hint] = decodeCCons - implicit lazy val decodeIwdrListingArrayEntry: Decoder[IwdrListingArrayEntry] = decodeCCons - implicit lazy val decodeMyriadInputInnerType: Decoder[MyriadInputInnerType] = decodeCCons - implicit lazy val decodeMyriadInputType: Decoder[MyriadInputType] = decodeCCons - implicit lazy val decodeMyriadOutputType: Decoder[MyriadOutputType] = decodeCCons - implicit lazy val decodeRequirement: Decoder[Requirement] = decodeCCons - implicit lazy val decodeResourceRequirementType: Decoder[ResourceRequirementType] = decodeCCons - implicit lazy val decodeSchemaDefTypes: Decoder[SchemaDefTypes] = decodeCCons - implicit lazy val decodeSecondaryFiles: Decoder[SecondaryFiles] = decodeCCons - implicit lazy val decodeSingleOrArrayOfStrings: Decoder[SingleOrArrayOfStrings] = decodeCCons - implicit lazy val decodeStringOrExpression: Decoder[StringOrExpression] = decodeCCons - - implicit lazy val encodeCwlAny: Encoder[CwlAny] = encodeCCons - implicit lazy val encodeExpression: Encoder[Expression] = encodeCCons - implicit lazy val encodeFileOrDirectory: Encoder[FileOrDirectory] = encodeCCons - implicit lazy val encodeGlob: Encoder[Glob] = encodeCCons - implicit lazy val encodeHint: Encoder[Hint] = encodeCCons - implicit lazy val encodeIwdrListingArrayEntry: Encoder[IwdrListingArrayEntry] = encodeCCons - implicit lazy val encodeMyriadInputInnerType: Encoder[MyriadInputInnerType] = encodeCCons - implicit lazy val encodeMyriadInputType: Encoder[MyriadInputType] = encodeCCons - implicit lazy val encodeMyriadOutputType: Encoder[MyriadOutputType] = encodeCCons - implicit lazy val encodeRequirement: Encoder[Requirement] = encodeCCons - implicit lazy val encodeResourceRequirementType: Encoder[ResourceRequirementType] = encodeCCons - implicit lazy val encodeSchemaDefTypes: Encoder[SchemaDefTypes] = encodeCCons - implicit lazy val encodeSecondaryFiles: Encoder[SecondaryFiles] = encodeCCons - implicit lazy val encodeSingleOrArrayOfStrings: Encoder[SingleOrArrayOfStrings] = encodeCCons - implicit lazy val encodeStringOrExpression: Encoder[StringOrExpression] = encodeCCons - - def decodeCwl(json: Json): Checked[Cwl] = { - findClass(json) match { - case Some("Workflow") => decodeWithErrorStringsJson[Workflow](json).map(Coproduct[Cwl].apply(_)) - case Some("CommandLineTool") => decodeWithErrorStringsJson[CommandLineTool](json).map(Coproduct[Cwl].apply(_)) - case Some("ExpressionTool") => decodeWithErrorStringsJson[ExpressionTool](json).map(Coproduct[Cwl].apply(_)) - case Some(other) => s"Class field was declared incorrectly: $other is not one of Workflow, CommandLineTool, or ExpressionTool! as seen in ${json.show}".invalidNelCheck - case None => s"Class field was omitted in ${json.show}".invalidNelCheck - } - } - - private def decodeWithErrorStringsJson[A](in: Json)(implicit d: Codec[A]): Checked[A] = - in.as[A].leftMap(_.show).leftMap(NonEmptyList.one) - - private def findClass(json: Json): Option[String] = - for { - obj <- json.asObject - map = obj.toMap - classObj <- map.get("class") - classString <- classObj.asString - } yield classString -} diff --git a/cwl/src/main/scala/cwl/CwlDecoder.scala b/cwl/src/main/scala/cwl/CwlDecoder.scala deleted file mode 100644 index b7a4125e5a3..00000000000 --- a/cwl/src/main/scala/cwl/CwlDecoder.scala +++ /dev/null @@ -1,84 +0,0 @@ -package cwl - -import better.files.{File => BFile} -import cats.data.EitherT._ -import cats.effect.IO -import cats.syntax.either._ -import cats.{Applicative, Monad} -import common.validation.ErrorOr._ -import common.validation.IOChecked._ -import common.validation.Validation._ -import cwl.preprocessor.{CwlFileReference, CwlPreProcessor, CwlReference} -import io.circe.Json - -import scala.util.Try - -object CwlDecoder { - - implicit val composedApplicative = Applicative[IO] compose Applicative[ErrorOr] - - def saladCwlFile(reference: CwlReference): IOChecked[String] = { - val cwlToolResult = - Try(CwltoolRunner.instance.salad(reference)) - .toCheckedWithContext(s"run cwltool on file ${reference.pathAsString}", throwableToStringFunction = t => t.toString) - - fromEither[IO](cwlToolResult) - } - - private lazy val cwlPreProcessor = new CwlPreProcessor() - - // TODO: WOM: During conformance testing the saladed-CWLs are referring to files in the temp directory. - // Thus we can't delete the temp directory until after the workflow is complete, like the workflow logs. - // All callers to this method should be fixed around the same time. - // https://github.com/broadinstitute/cromwell/issues/3186 - def todoDeleteCwlFileParentDirectory(cwlFile: BFile): IOChecked[Unit] = { - goIOChecked { - //cwlFile.parent.delete(swallowIOExceptions = true) - } - } - - def parseJson(json: Json, from: String): IOChecked[Cwl] = fromEither[IO](CwlCodecs.decodeCwl(json).contextualizeErrors(s"parse '$from'")) - - /** - * Notice it gives you one instance of Cwl. This has transformed all embedded files into scala object state - */ - def decodeCwlReference(reference: CwlReference)(implicit processor: CwlPreProcessor = cwlPreProcessor): IOChecked[Cwl] = { - def makeStandaloneWorkflow(): IOChecked[Json] = processor.preProcessCwl(reference) - - for { - standaloneWorkflow <- makeStandaloneWorkflow() - parsedCwl <- parseJson(standaloneWorkflow, reference.pathAsString) - } yield parsedCwl - } - - def decodeCwlFile(file: BFile, workflowRoot: Option[String] = None) = { - decodeCwlReference(CwlFileReference(file, workflowRoot)) - } - - def decodeCwlString(cwl: String, - zipOption: Option[BFile] = None, - rootName: Option[String] = None, - cwlFilename: String = "cwl_temp_file"): IOChecked[Cwl] = { - for { - parentDir <- goIOChecked(BFile.newTemporaryDirectory("cwl_temp_dir_")) // has a random long appended like `cwl_temp_dir_100000000000` - file <- fromEither[IO](parentDir./(cwlFilename + ".cwl").write(cwl).asRight) // serves as the basis for the output directory name; must remain stable across restarts - _ <- zipOption match { - case Some(zip) => goIOChecked(zip.unzipTo(parentDir)) - case None => Monad[IOChecked].unit - } - out <- decodeCwlFile(file, rootName) - _ <- todoDeleteCwlFileParentDirectory(file) - } yield out - } - - //This is used when traversing over Cwl and replacing links w/ embedded data - private[cwl] def decodeCwlAsValidated(fileName: String): IOCheckedValidated[(String, Cwl)] = { - //The SALAD preprocess step puts "file://" as a prefix to all filenames. Better files doesn't like this. - val bFileName = fileName.stripPrefix("file://") - - decodeCwlReference(CwlFileReference(BFile(bFileName), None)). - map(fileName.toString -> _). - value. - map(_.toValidated) - } -} diff --git a/cwl/src/main/scala/cwl/CwlExecutableValidation.scala b/cwl/src/main/scala/cwl/CwlExecutableValidation.scala deleted file mode 100644 index f6f8237c5ff..00000000000 --- a/cwl/src/main/scala/cwl/CwlExecutableValidation.scala +++ /dev/null @@ -1,38 +0,0 @@ -package cwl - -import common.Checked -import common.validation.Checked._ -import io.circe.Json -import wom.callable.{ExecutableCallable, TaskDefinition} -import wom.executable.Executable -import wom.executable.Executable.{InputParsingFunction, ParsedInputMap} -import wom.expression.IoFunctionSet -import wom.util.YamlUtils - -object CwlExecutableValidation { - - // Decodes the input file, and build the ParsedInputMap - private val inputCoercionFunction: InputParsingFunction = - inputFile => { - val yaml = YamlUtils.parse(inputFile) - yaml.flatMap(_.as[Map[String, Json]]) match { - case Left(error) => error.getMessage.invalidNelCheck[ParsedInputMap] - case Right(inputValue) => inputValue.map({ case (key, value) => key -> value.foldWith(CwlJsonToDelayedCoercionFunction) }).validNelCheck - } - } - - def buildWomExecutableCallable(callable: Checked[ExecutableCallable], inputFile: Option[String], ioFunctions: IoFunctionSet, strictValidation: Boolean): Checked[Executable] = { - for { - womDefinition <- callable - executable <- Executable.withInputs(womDefinition, inputCoercionFunction, inputFile, ioFunctions, strictValidation) - } yield executable - } - - def buildWomExecutable(callableTaskDefinition: Checked[TaskDefinition], inputFile: Option[String], ioFunctions: IoFunctionSet, strictValidation: Boolean): Checked[Executable] = { - for { - taskDefinition <- callableTaskDefinition - executableTaskDefinition = taskDefinition.toExecutable.toEither - executable <- CwlExecutableValidation.buildWomExecutableCallable(executableTaskDefinition, inputFile, ioFunctions, strictValidation) - } yield executable - } -} diff --git a/cwl/src/main/scala/cwl/CwlExpressionCommandPart.scala b/cwl/src/main/scala/cwl/CwlExpressionCommandPart.scala deleted file mode 100644 index f323c74dbd2..00000000000 --- a/cwl/src/main/scala/cwl/CwlExpressionCommandPart.scala +++ /dev/null @@ -1,146 +0,0 @@ -package cwl - -import cats.syntax.either._ -import cats.syntax.validated._ -import common.Checked -import common.validation.Checked._ -import common.validation.ErrorOr.ErrorOr -import wom.callable.RuntimeEnvironment -import wom.expression.IoFunctionSet -import wom.graph.LocalName -import wom.values._ -import wom.{CommandPart, InstantiatedCommand} - -import scala.language.postfixOps - -case class CwlExpressionCommandPart(expr: Expression)(hasShellCommandRequirement: Boolean, expressionLib: ExpressionLib) extends CommandPart { - override def instantiate(inputsMap: Map[LocalName, WomValue], - functions: IoFunctionSet, - valueMapper: (WomValue) => WomValue, - runtimeEnvironment: RuntimeEnvironment): ErrorOr[List[InstantiatedCommand]] = { - val stringKeyMap = inputsMap map { - case (LocalName(localName), value) => localName -> valueMapper(value) - } - val parameterContext = ParameterContext( - functions, - expressionLib, - inputs = stringKeyMap, runtimeOption = Option(runtimeEnvironment) - ) - ExpressionEvaluator.eval(expr, parameterContext) map { womValue => - List(InstantiatedCommand(valueMapper(womValue).valueString.shellQuote)) - } - } -} - -/** - * Generates command parts from a CWL Binding - */ -abstract class CommandLineBindingCommandPart(commandLineBinding: CommandLineBinding)(hasShellCommandRequirement: Boolean, expressionLib: ExpressionLib) extends CommandPart { - - - private lazy val prefixAsString = commandLineBinding.prefix.getOrElse("") - private lazy val prefixAsList = commandLineBinding.prefix.toList - private lazy val separate = commandLineBinding.effectiveSeparate - - def handlePrefix(value: String) = { - if (separate) prefixAsList :+ value else List(s"$prefixAsString$value") - } - - /** - * Value bound to this command part. - * InputCommandLineBindingCommandPart has one - * ArgumentCommandLineBindingCommandPart does not - */ - def boundValue: Option[WomValue] - - // If the bound value is defined but contains an empty optional value, we should not evaluate the valueFrom - // Conformance test "stage-unprovided-file" tests this behavior - private lazy val evaluateValueFrom = boundValue.forall { - case WomOptionalValue(_, None) => false - case _ => true - } - - override def instantiate(inputsMap: Map[LocalName, WomValue], - functions: IoFunctionSet, - valueMapper: (WomValue) => WomValue, - runtimeEnvironment: RuntimeEnvironment): ErrorOr[List[InstantiatedCommand]] = { - val stringInputsMap = inputsMap map { - case (LocalName(localName), value) => localName -> valueMapper(value) - } - val parameterContext = ParameterContext( - functions, - expressionLib, - inputs = stringInputsMap, - runtimeOption = Option(runtimeEnvironment), - self = boundValue.getOrElse(ParameterContext.EmptySelf) - ) - - val evaluatedValueFrom = commandLineBinding.optionalValueFrom flatMap { - case StringOrExpression.Expression(expression) if evaluateValueFrom => Option(ExpressionEvaluator.eval(expression, parameterContext) map valueMapper) - case StringOrExpression.String(string) if evaluateValueFrom => Option(WomString(string).validNel) - case _ => None - } - - val evaluatedWomValue: Checked[WomValue] = evaluatedValueFrom.orElse(boundValue.map(_.validNel)) match { - case Some(womValue) => womValue.map(valueMapper).toEither - case None => "Command line binding has no valueFrom field and no bound value".invalidNelCheck - } - - def applyShellQuote(value: String): String = commandLineBinding.shellQuote match { - // Only honor shellQuote = false if ShellCommandRequirement is enabled. - // Conformance test "Test that shell directives are not interpreted." - case Some(false) if hasShellCommandRequirement => value - case _ => value.shellQuote - } - - def processValue(womValue: WomValue): List[String] = womValue match { - case WomOptionalValue(_, Some(value)) => processValue(valueMapper(value)) - case WomOptionalValue(_, None) => List.empty - case _: WomString | _: WomInteger | _: WomFile | _: WomLong | _: WomFloat => handlePrefix(valueMapper(womValue).valueString) - // For boolean values, use the value of the boolean to choose whether to print the prefix or not - case WomBoolean(false) => List.empty - case WomBoolean(true) => prefixAsList - case WomArray(_, values) => commandLineBinding.itemSeparator match { - case Some(_) if values.isEmpty => List.empty - case Some(itemSeparator) => handlePrefix(values.map(valueMapper(_).valueString).mkString(itemSeparator)) - - /* - via: http://www.commonwl.org/v1.0/CommandLineTool.html#CommandLineBinding - - > If itemSeparator is specified, add prefix and the join the array into a single string with itemSeparator - > separating the items. Otherwise first add prefix, then recursively process individual elements. - - Not 100% sure if this is conformant, as we are only recurse the head into `processValue` here... However the - conformance test "Test InlineJavascriptRequirement with multiple expressions in the same tool" is happy with the - behavior. - */ - // InstantiatedCommand elements are generated here when there exists an optionalValueFrom - case None if commandLineBinding.optionalValueFrom.isDefined => values.toList match { - case head :: tail => processValue(head) ++ tail.map(valueMapper(_).valueString) - case Nil => prefixAsList - } - - // When there is no optionalValueFrom the InstantiatedCommand elements for the womValue are appended elsewhere - case _ => prefixAsList - } - case _: WomObjectLike => prefixAsList - case WomEnumerationValue(_, value) => handlePrefix(value) - case WomCoproductValue(_, value) => processValue(value) - case w => throw new RuntimeException(s"Unhandled CwlExpressionCommandPart value '$w' of type ${w.womType.stableName}") - } - - evaluatedWomValue map { v => processValue(v) map applyShellQuote map (InstantiatedCommand(_)) } toValidated - } -} - -case class InputCommandLineBindingCommandPart(commandLineBinding: InputCommandLineBinding, associatedValue: WomValue) - (hasShellCommandRequirement: Boolean, expressionLib: ExpressionLib) - extends CommandLineBindingCommandPart(commandLineBinding)(hasShellCommandRequirement, expressionLib) { - override lazy val boundValue = Option(associatedValue) -} - -case class ArgumentCommandLineBindingCommandPart(commandLineBinding: ArgumentCommandLineBinding) - (hasShellCommandRequirement: Boolean, expressionLib: ExpressionLib) - extends CommandLineBindingCommandPart(commandLineBinding)(hasShellCommandRequirement, expressionLib) { - override lazy val boundValue = None -} diff --git a/cwl/src/main/scala/cwl/CwlJsonToDelayedCoercionFunction.scala b/cwl/src/main/scala/cwl/CwlJsonToDelayedCoercionFunction.scala deleted file mode 100644 index 55dc7f9e3ba..00000000000 --- a/cwl/src/main/scala/cwl/CwlJsonToDelayedCoercionFunction.scala +++ /dev/null @@ -1,90 +0,0 @@ -package cwl - -import cats.syntax.option._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import common.validation.ErrorOr.ErrorOr -import common.validation.Validation._ -import cwl.CwlCodecs._ -import io.circe.{Json, JsonNumber, JsonObject} -import wom.executable.Executable.DelayedCoercionFunction -import wom.types._ -import wom.values._ - -// With recursive types we could let circe parse it for us, but until we figure that out just parse it as Json and -// manually check for File / Directory structures -private [cwl] object CwlJsonToDelayedCoercionFunction extends Json.Folder[DelayedCoercionFunction] { - private def simpleCoercion(value: Any)(womType: WomType): ErrorOr[WomValue] = womType.coerceRawValue(value).toErrorOr - - override def onNull: DelayedCoercionFunction = WomOptionalValue.none(_).validNel - - override def onBoolean(value: Boolean): DelayedCoercionFunction = simpleCoercion(value) - - override def onNumber(value: JsonNumber): DelayedCoercionFunction = { - case WomFloatType => WomFloat(value.toDouble).validNel - case WomIntegerType => value.toInt.map(WomInteger.apply).toValidNel(s"$value is not a valid Int") - case WomLongType => value.toLong.map(WomLong.apply).toValidNel(s"$value is not a valid Long") - case other => other.coerceRawValue(value.toString).toErrorOr - } - - override def onString(value: String): DelayedCoercionFunction = simpleCoercion(value) - - override def onArray(value: Vector[Json]): DelayedCoercionFunction = { - case womArrayType: WomArrayType => - value.toList - .traverse(_.foldWith(this).apply(womArrayType.memberType)) - .map { - WomArray(womArrayType, _) - } - - case WomOptionalType(otherType) => - onArray(value).apply(otherType) - case WomCoproductType(types) => - val attempts: List[ErrorOr[WomValue]] = types.toList.map(onArray(value)(_)) - attempts.find(_.isValid).getOrElse(attempts.sequence[ErrorOr, WomValue].map(_.head)) - case WomAnyType => - // Make an array of WomAny - value.toList - .traverse(_.foldWith(this).apply(WomAnyType)) - .map { - WomArray(WomArrayType(WomAnyType), _) - } - case other => s"Cannot convert an array input value into a non array type: $other".invalidNel - } - - override def onObject(value: JsonObject): DelayedCoercionFunction = { - // CWL files are represented as Json objects, so this could be a file - case WomSingleFileType | WomMaybePopulatedFileType if value.toMap.get("class").flatMap(_.asString).contains("File") => - Json.fromJsonObject(value).as[File] match { - case Left(errors) => errors.message.invalidNel - case Right(file) => file.asWomValue - } - case WomMaybeListedDirectoryType | WomUnlistedDirectoryType if value.toMap.get("class").flatMap(_.asString).contains("Directory") => - Json.fromJsonObject(value).as[Directory] match { - case Left(errors) => errors.message.invalidNel - case Right(directory) => directory.asWomValue - } - case composite: WomCompositeType => - val foldedMap = value.toList.traverse[ErrorOr, (String, WomValue)]({ - case (k, v) => - composite.typeMap.get(k).map({ valueType => - v.foldWith(this).apply(valueType).map(k -> _) - }).getOrElse(s"Input field $k is not defined in the composite input type $composite".invalidNel) - }).map(_.toMap[String, WomValue]) - - foldedMap.map(WomObject.withTypeUnsafe(_, composite)) - case WomObjectType => - val foldedMap = value.toList.traverse[ErrorOr, (String, WomValue)]({ - case (k, v) => v.foldWith(this).apply(WomAnyType).map(k -> _) - }).map(_.toMap[String, WomValue]) - - foldedMap.map(WomObject.apply) - case WomOptionalType(otherType) => onObject(value).apply(otherType) - case WomCoproductType(types) => - val attempts: List[ErrorOr[WomValue]] = types.toList.map(onObject(value)(_)) - //these are all Invalid, just taking head to satisfy required type of WomValue instead of List[WomValue] - attempts.find(_.isValid).getOrElse(attempts.sequence[ErrorOr, WomValue].map(_.head)) - case other => s"Cannot convert an object value $value into a non array type: $other".invalidNel - } -} diff --git a/cwl/src/main/scala/cwl/CwlToRun.scala b/cwl/src/main/scala/cwl/CwlToRun.scala deleted file mode 100644 index 1571958abb5..00000000000 --- a/cwl/src/main/scala/cwl/CwlToRun.scala +++ /dev/null @@ -1,11 +0,0 @@ -package cwl - -import shapeless._ - -object CwlToRun extends Poly1 { - implicit def commandLineTool = at[CommandLineTool] { Coproduct[WorkflowStep.Run](_) } - implicit def workflow = at[Workflow] { Coproduct[WorkflowStep.Run](_) } - implicit def expressionTool = at[ExpressionTool] { Coproduct[WorkflowStep.Run](_) } -} - - diff --git a/cwl/src/main/scala/cwl/CwlType.scala b/cwl/src/main/scala/cwl/CwlType.scala deleted file mode 100644 index 7ec171d97fb..00000000000 --- a/cwl/src/main/scala/cwl/CwlType.scala +++ /dev/null @@ -1,284 +0,0 @@ -package cwl - -import cats.instances.list._ -import cats.instances.option._ -import cats.syntax.functor._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import common.validation.ErrorOr._ -import common.validation.IOChecked._ -import eu.timepit.refined._ -import mouse.all._ -import shapeless.Poly1 -import shapeless.syntax.singleton._ -import wom.expression.IoFunctionSet.{IoDirectory, IoFile} -import wom.expression.{IoFunctionSet, PathFunctionSet} -import wom.types.WomFileType -import wom.values._ - -import scala.annotation.tailrec - -object CwlType extends Enumeration { - type CwlType = Value - - val Any = Value("Any") - val Null = Value("null") - val Boolean = Value("boolean") - val Int = Value("int") - val Long = Value("long") - val Float = Value("float") - val Double = Value("double") - val String = Value("string") - val File = Value("File") - val Directory = Value("Directory") -} - -case class File private -( - `class`: W.`"File"`.T, - location: Option[String], //TODO refine w/ regex of IRI - path: Option[String], - basename: Option[String], - checksum: Option[String], - size: Option[Long], - secondaryFiles: Option[Array[FileOrDirectory]], - format: Option[String], - contents: Option[String] -) { - lazy val effectivePath = path.orElse(location) - - lazy val errorOrSecondaryFiles: ErrorOr[List[WomFile]] = { - val dirsOrFiles: List[FileOrDirectory] = secondaryFiles.getOrElse(Array.empty).toList - dirsOrFiles.traverse{ - _.fold(CwlDirectoryOrFileAsWomSingleDirectoryOrFile) - } - } - - lazy val asWomValue: ErrorOr[WomMaybePopulatedFile] = { - errorOrSecondaryFiles flatMap { secondaryFiles => - val valueOption = location.orElse(path) - (valueOption, contents) match { - case (None, None) => - "Cannot convert CWL File to WomValue without either a location, a path, or contents".invalidNel - case (None, Some(content)) => - val initializeFunction: WomMaybePopulatedFile => IoFunctionSet => IOChecked[WomValue] = { file =>ioFunctionSet => - val name = basename.getOrElse(content.hashCode.toString) - ioFunctionSet.writeFile(name, content).toIOChecked(ioFunctionSet.cs) map { writtenFile => - file.copy(valueOption = Option(writtenFile.value)) - } - } - - new WomMaybePopulatedFile(None, checksum, size, format, contents, initializeFunction = initializeFunction).valid - case (_, _) => - WomMaybePopulatedFile(valueOption, checksum, size, format, contents, secondaryFiles).valid - } - } - } -} - -object File { - def apply( - location: Option[String] = None, //TODO refine w/ regex of IRI - path: Option[String] = None, - basename: Option[String] = None, - checksum: Option[String] = None, - size: Option[Long] = None, - secondaryFiles: Option[Array[FileOrDirectory]] = None, - format: Option[String] = None, - contents: Option[String] = None): File = { - new cwl.File( - "File".narrow, - location, - path, - basename, - checksum, - size, - secondaryFiles, - format, - contents - ) - } - - def dirname(value: String): String = { - val index = value.lastIndexOf('/') - if (index >= 0) { - value.substring(0, index) - } else { - "" - } - } - - def basename(value: String): String = value.substring(value.lastIndexOf('/') + 1) - - def nameroot(value: String): String = basename(value).stripSuffix(nameext(value)) - - def nameext(value: String): String = { - val base = basename(value) - val index = base.lastIndexOf('.') - if (index >= 0) { - base.substring(index) - } else { - "" - } - } - - def recursivelyBuildDirectory(directory: String, ioFunctions: IoFunctionSet)(visited: Vector[String] = Vector.empty): IOChecked[WomMaybeListedDirectory] = { - for { - listing <- ioFunctions.listDirectory(directory)(visited).toIOChecked(ioFunctions.cs) - fileListing <- listing.toList.traverse[IOChecked, WomFile]({ - case IoDirectory(e) => recursivelyBuildDirectory(e, ioFunctions)(visited :+ directory).widen - case IoFile(e) => WomMaybePopulatedFile(e).validIOChecked.widen - }) - } yield WomMaybeListedDirectory(Option(directory), Option(fileListing)) - } - - private def asAbsoluteSiblingOfPrimary(primary: WomFile, pathFunctions: PathFunctionSet)(path: String) = { - pathFunctions.absoluteSibling(primary.value, path) - } - - def secondaryStringFile(primaryWomFile: WomFile, - stringWomFileType: WomFileType, - secondaryValue: String, - ioFunctions: IoFunctionSet): IOChecked[WomFile] = { - val secondaryRelativeFileName = File.relativeFileName(primaryWomFile.value, secondaryValue) - - // If the primary file is an absolute path, and the secondary is not, make the secondary file an absolute path and a sibling of the primary - // http://www.commonwl.org/v1.0/CommandLineTool.html#CommandInputParameter - val filePath = asAbsoluteSiblingOfPrimary(primaryWomFile, ioFunctions.pathFunctions)(secondaryRelativeFileName) - - // If the secondary file is in fact a directory, look into it and build its listing - for { - isDirectory <- ioFunctions.isDirectory(filePath).toIOChecked(ioFunctions.cs) - file <- if (isDirectory) recursivelyBuildDirectory(filePath, ioFunctions)() else WomFile(stringWomFileType, filePath).validIOChecked - } yield file - } - - def secondaryExpressionFiles(primaryWomFile: WomFile, - stringWomFileType: WomFileType, - expression: Expression, - parameterContext: ParameterContext, - expressionLib: ExpressionLib, - ioFunctions: IoFunctionSet): ErrorOr[List[WomFile]] = { - - /* - If the value is an expression, the value of self in the expression must be the primary input or output File object - to which this binding applies. - */ - val secondaryParameterContext = parameterContext.copy(self = primaryWomFile) - - /* - The expression must return a filename string relative to the path to the primary File, a File or Directory object - with either path or location and basename fields set, or an array consisting of strings or File or Directory - objects. - */ - def parseResult(nestedLevel: Int)(womValue: WomValue): ErrorOr[List[WomFile]] = { - womValue match { - case womString: WomString => - List(WomFile(stringWomFileType, womString.value |> asAbsoluteSiblingOfPrimary(primaryWomFile, ioFunctions.pathFunctions))).valid - case womMaybeListedDirectory: WomMaybeListedDirectory => - List(womMaybeListedDirectory.mapFile(asAbsoluteSiblingOfPrimary(primaryWomFile, ioFunctions.pathFunctions))).valid - case womMaybePopulatedFile: WomMaybePopulatedFile => - List(womMaybePopulatedFile.mapFile(asAbsoluteSiblingOfPrimary(primaryWomFile, ioFunctions.pathFunctions))).valid - case womArray: WomArray if nestedLevel == 0 => - womArray.value.toList flatTraverse parseResult(nestedLevel + 1) - case other => s"Not a valid secondary file: $other".invalidNel - } - } - - val possibleArrayErrorOr: ErrorOr[WomValue] = ExpressionEvaluator.eval(expression, secondaryParameterContext) - possibleArrayErrorOr.flatMap(parseResult(nestedLevel = 0)) - } - - def relativeFileName(primary: String, secondary: String): String = { - /* - If a value in secondaryFiles is a string that is not an expression, it specifies that the following pattern should - be applied to the path of the primary file to yield a filename relative to the primary File: - - 1. If string begins with one or more caret ^ characters, for each caret, remove the last file extension from the - path (the last period . and all following characters). If there are no file extensions, the path is unchanged. - - 2. Append the remainder of the string to the end of the file path. - */ - if (secondary.startsWith("^")) { - @tailrec - def stripCaret(primaryAcc: String, secondaryAcc: String): (String, String) = { - if (secondaryAcc.startsWith("^")) { - val idx = primaryAcc.lastIndexOf('.') - if (idx < 0) { - (primaryAcc, secondaryAcc.dropWhile(_ == '^')) - } else { - val primaryNext = primaryAcc.substring(0, idx) - val secondaryNext = secondaryAcc.drop(1) - stripCaret(primaryNext, secondaryNext) - } - } else { - (primaryAcc, secondaryAcc) - } - } - - val (prefix, suffix) = stripCaret(primary, secondary) - prefix + suffix - } else { - primary + secondary - } - - } -} - -case class Directory private -( - `class`: W.`"Directory"`.T, - location: Option[String], - path: Option[String], - basename: Option[String], - listing: Option[Array[FileOrDirectory]] -) { - lazy val errorOrListingOption: ErrorOr[Option[List[WomFile]]] = { - val maybeErrorOrList: Option[ErrorOr[List[WomFile]]] = - listing map { - _.toList.traverse { - _.fold(CwlDirectoryOrFileAsWomSingleDirectoryOrFile) - } - } - maybeErrorOrList.sequence[ErrorOr, List[WomFile]] - } - - lazy val asWomValue: ErrorOr[WomFile] = { - errorOrListingOption flatMap { listingOption => - path.orElse(location) map { value => - WomMaybeListedDirectory(Option(value), listingOption, basename).valid - } getOrElse { - val initializeFunction: WomMaybeListedDirectory => IoFunctionSet => IOChecked[WomValue] = { dir =>ioFunctionSet => - ioFunctionSet.createTemporaryDirectory(basename).toIOChecked(ioFunctionSet.cs) map { tempDir => - dir.copy(valueOption = Option(tempDir)) - } - } - new WomMaybeListedDirectory(None, listingOption, basename, initializeFunction = initializeFunction).valid - } - } - } -} - -object Directory { - def apply(location: Option[String] = None, - path: Option[String] = None, - basename: Option[String] = None, - listing: Option[Array[FileOrDirectory]] = None - ): Directory = - new cwl.Directory("Directory".narrow, location, path, basename, listing) - - def basename(value: String): String = { - val stripped = value.stripSuffix("/") - stripped.substring(stripped.lastIndexOf('/') + 1) - } -} - -private[cwl] object CwlDirectoryOrFileAsWomSingleDirectoryOrFile extends Poly1 { - implicit def caseFile: Case.Aux[File, ErrorOr[WomFile]] = at { - _.asWomValue - } - - implicit def caseDirectory: Case.Aux[Directory, ErrorOr[WomFile]] = at { - _.asWomValue - } -} diff --git a/cwl/src/main/scala/cwl/CwlVersion.scala b/cwl/src/main/scala/cwl/CwlVersion.scala deleted file mode 100644 index 12336b1cceb..00000000000 --- a/cwl/src/main/scala/cwl/CwlVersion.scala +++ /dev/null @@ -1,7 +0,0 @@ -package cwl - -object CwlVersion extends Enumeration { - type CwlVersion = Value - - val Version1 = Value("v1.0") -} diff --git a/cwl/src/main/scala/cwl/CwlWomExpression.scala b/cwl/src/main/scala/cwl/CwlWomExpression.scala deleted file mode 100644 index 5d77370ba49..00000000000 --- a/cwl/src/main/scala/cwl/CwlWomExpression.scala +++ /dev/null @@ -1,260 +0,0 @@ -package cwl - -import cats.instances.list._ -import cats.syntax.functor._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import common.validation.ErrorOr.{ErrorOr, ShortCircuitingFlatMap} -import common.validation.IOChecked._ -import common.validation.Validation._ -import cwl.ExpressionEvaluator.{ECMAScriptExpression, ECMAScriptFunction} -import cwl.InitialWorkDirFileGeneratorExpression._ -import cwl.InitialWorkDirRequirement.IwdrListingArrayEntry -import shapeless.Poly1 -import wom.callable.{AdHocValue, ContainerizedInputExpression} -import wom.expression.IoFunctionSet.{IoDirectory, IoFile} -import wom.expression.{FileEvaluation, IoFunctionSet, WomExpression} -import wom.types._ -import wom.values._ - -import scala.concurrent.Await -import scala.concurrent.duration.Duration - -trait CwlWomExpression extends WomExpression { - - def cwlExpressionType: WomType - - override def evaluateType(inputTypes: Map[String, WomType]): ErrorOr[WomType] = cwlExpressionType.validNel - - def expressionLib: ExpressionLib - - def evaluate(inputs: Map[String, WomValue], parameterContext: ParameterContext, expression: Expression): ErrorOr[WomValue] = - expression. - fold(EvaluateExpression). - apply(parameterContext) -} - -case class ECMAScriptWomExpression(expression: Expression, - override val inputs: Set[String], - override val expressionLib: ExpressionLib) extends CwlWomExpression { - val cwlExpressionType = WomAnyType - - override def sourceString = expression match { - case Expression.ECMAScriptExpression(s) => s.value - case Expression.ECMAScriptFunction(s) => s.value - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet) = { - val pc = ParameterContext(ioFunctionSet, expressionLib, inputValues) - evaluate(inputValues, pc, expression) - } - - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType) = Set.empty[FileEvaluation].validNel -} - -final case class InitialWorkDirFileGeneratorExpression(entry: IwdrListingArrayEntry, expressionLib: ExpressionLib) extends ContainerizedInputExpression { - - def evaluate(inputValues: Map[String, WomValue], mappedInputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): IOChecked[List[AdHocValue]] = { - def recursivelyBuildDirectory(directory: String): IOChecked[WomMaybeListedDirectory] = { - import cats.syntax.traverse._ - for { - listing <- ioFunctionSet.listDirectory(directory)().toIOChecked(ioFunctionSet.cs) - fileListing <- listing.toList.traverse[IOChecked, WomFile] { - case IoDirectory(e) => recursivelyBuildDirectory(e).widen - case IoFile(e) => WomSingleFile(e).validIOChecked.widen - } - } yield WomMaybeListedDirectory(Option(directory), Option(fileListing)) - } - - inputValues.toList.traverse[IOChecked, (String, WomValue)]({ - case (k, v: WomMaybeListedDirectory) => - val absolutePathString = ioFunctionSet.pathFunctions.relativeToHostCallRoot(v.value) - recursivelyBuildDirectory(absolutePathString).contextualizeErrors(s"Error building directory $absolutePathString") map { k -> _ } - case kv => kv.validIOChecked - }).map(_.toMap) - .flatMap({ updatedValues => - val unmappedParameterContext = ParameterContext(ioFunctionSet, expressionLib, updatedValues) - entry.fold(InitialWorkDirFilePoly).apply(unmappedParameterContext, mappedInputValues).toIOChecked - }) - } -} - -object InitialWorkDirFileGeneratorExpression { - type InitialWorkDirFileEvaluator = (ParameterContext, Map[String, WomValue]) => ErrorOr[List[AdHocValue]] - - /** - * Converts an InitialWorkDir. - * - * TODO: Review against the spec. Especially for Dirent values. For example: - * - * "If the value is an expression that evaluates to a Dirent object, this indicates that the File or Directory in - * entry should be added to the designated output directory with the name in entryname." - * - * - http://www.commonwl.org/v1.0/CommandLineTool.html#InitialWorkDirRequirement - * - http://www.commonwl.org/v1.0/CommandLineTool.html#Dirent - */ - object InitialWorkDirFilePoly extends Poly1 { - implicit val caseExpressionDirent: Case.Aux[ExpressionDirent, InitialWorkDirFileEvaluator] = { - at { expressionDirent => - (unmappedParameterContext, mappedInputValues) => { - - val mutableInputOption = expressionDirent.entry.fold(ExpressionToMutableInputOptionPoly) - - val entryEvaluation = - expressionDirent.entry match { - //we need to catch this special case to feed in "value-mapped" input values - case expr@Expression.ECMAScriptExpression(exprString) if exprString.value.trim() == "$(JSON.stringify(inputs))" => - val specialParameterContext = ParameterContext( - unmappedParameterContext.ioFunctionSet, - unmappedParameterContext.expressionLib, - mappedInputValues - ) - ExpressionEvaluator.eval(expr, specialParameterContext) - case _ => ExpressionEvaluator.eval(expressionDirent.entry, unmappedParameterContext) - } - - val womValueErrorOr: ErrorOr[AdHocValue] = entryEvaluation flatMap { - case womFile: WomFile => - val errorOrEntryName: ErrorOr[Option[String]] = expressionDirent.entryname match { - case Some(actualEntryName) => actualEntryName.fold(EntryNamePoly).apply(unmappedParameterContext).map(Option.apply) - case None => None.valid - } - errorOrEntryName map { entryName => - AdHocValue(womFile, entryName, inputName = mutableInputOption) - } - case other => for { - coerced <- WomStringType.coerceRawValue(other).toErrorOr - contentString = coerced.asInstanceOf[WomString].value - // We force the entryname to be specified, and then evaluate it: - entryNameStringOrExpression <- expressionDirent.entryname.toErrorOr( - "Invalid dirent: Entry was a string but no file name was supplied") - entryName <- entryNameStringOrExpression.fold(EntryNamePoly).apply(unmappedParameterContext) - writeFile = unmappedParameterContext.ioFunctionSet.writeFile(entryName, contentString) - writtenFile <- validate(Await.result(writeFile, Duration.Inf)) - } yield AdHocValue(writtenFile, alternativeName = None, inputName = mutableInputOption) - } - - womValueErrorOr.map(List(_)) - } - } - } - - implicit val caseStringDirent: Case.Aux[StringDirent, InitialWorkDirFileEvaluator] = { - at { - stringDirent => { - (unmappedParameterContext, _) => - val womValueErrorOr = for { - entryName <- stringDirent.entryname.fold(EntryNamePoly).apply(unmappedParameterContext) - contentString = stringDirent.entry - writeFile = unmappedParameterContext.ioFunctionSet.writeFile(entryName, contentString) - writtenFile <- validate(Await.result(writeFile, Duration.Inf)) - } yield writtenFile - - womValueErrorOr.map(AdHocValue(_, alternativeName = None, inputName = None)).map(List(_)) - } - } - } - - implicit val caseExpression: Case.Aux[Expression, InitialWorkDirFileEvaluator] = { - at { expression => - (unmappedParameterContext, _) => { - // A single expression which must evaluate to an array of Files - val expressionEvaluation = ExpressionEvaluator.eval(expression, unmappedParameterContext) - - expressionEvaluation flatMap { - case array: WomArray if array.value.forall(_.isInstanceOf[WomFile]) => - array.value.toList.map(_.asInstanceOf[WomFile]).map(AdHocValue(_, alternativeName = None, inputName = None)).validNel - case file: WomFile => - List(AdHocValue(file, alternativeName = None, inputName = None)).validNel - case other => - val error = "InitialWorkDirRequirement listing expression must be File or Array[File] but got %s: %s" - .format(other, other.womType.stableName) - error.invalidNel - } - } - } - } - - implicit val caseString: Case.Aux[String, InitialWorkDirFileEvaluator] = { - at { string => - (_, _) => { - List(AdHocValue(WomSingleFile(string), alternativeName = None, inputName = None)).valid - } - } - } - - implicit val caseStringOrExpression: Case.Aux[StringOrExpression, InitialWorkDirFileEvaluator] = { - at { - _.fold(this) - } - } - - implicit val caseFile: Case.Aux[File, InitialWorkDirFileEvaluator] = { - at { file => - (_, _) => { - file.asWomValue.map(AdHocValue(_, alternativeName = None, inputName = None)).map(List(_)) - } - } - } - - implicit val caseDirectory: Case.Aux[Directory, InitialWorkDirFileEvaluator] = { - at { directory => - (_, _) => { - directory.asWomValue.map(AdHocValue(_, alternativeName = None, inputName = None)).map(List(_)) - } - } - } - - } - - type EntryNameEvaluator = ParameterContext => ErrorOr[String] - - object EntryNamePoly extends Poly1 { - implicit val caseString: Case.Aux[String, EntryNameEvaluator] = { - at { - string => { - _ => - string.valid - } - } - } - - implicit val caseExpression: Case.Aux[Expression, EntryNameEvaluator] = { - at { - expression => { - parameterContext => - for { - entryNameExpressionEvaluated <- ExpressionEvaluator.eval(expression, parameterContext) - entryNameValidated <- mustBeString(entryNameExpressionEvaluated) - } yield entryNameValidated - } - } - } - - private def mustBeString(womValue: WomValue): ErrorOr[String] = { - womValue match { - case WomString(s) => s.valid - case other => WomStringType.coerceRawValue(other).map(_.asInstanceOf[WomString].value).toErrorOr - } - } - } - - /** - * Searches for ECMAScript expressions that mutate input paths. - * - * @see [[AdHocValue]] - */ - object ExpressionToMutableInputOptionPoly extends Poly1 { - private val MutableInputRegex = """(?s)\s*\$\(\s*inputs\.(\w+)\s*\)\s*""".r - implicit val caseECMAScriptExpression: Case.Aux[ECMAScriptExpression, Option[String]] = { - at { eCMAScriptExpression => - eCMAScriptExpression.value match { - case MutableInputRegex(mutableInput) => Option(mutableInput) - case _ => None - } - } - } - implicit val caseECMAScriptFunction: Case.Aux[ECMAScriptFunction, Option[String]] = at { _ => None } - } -} diff --git a/cwl/src/main/scala/cwl/CwltoolRunner.scala b/cwl/src/main/scala/cwl/CwltoolRunner.scala deleted file mode 100644 index 834c9d80b23..00000000000 --- a/cwl/src/main/scala/cwl/CwltoolRunner.scala +++ /dev/null @@ -1,71 +0,0 @@ -package cwl - -import ammonite.ops.ImplicitWd._ -import ammonite.ops._ -import com.typesafe.config.ConfigFactory -import cwl.preprocessor.CwlReference -import org.broadinstitute.heterodon.ExecAndEval - -/** - * Interface for running cwltool. - */ -sealed trait CwltoolRunner { - def salad(reference: CwlReference): String -} - -object CwltoolRunner { - private lazy val config = ConfigFactory.load - - lazy val instance: CwltoolRunner = { - val runnerClass = config.getString("cwltool-runner.class") - Class.forName(runnerClass).getDeclaredConstructor().newInstance().asInstanceOf[CwltoolRunner] - } -} - -/** - * Runs cwltool as an external process. - */ -final class CwltoolProcess extends CwltoolRunner { - override def salad(reference: CwlReference): String = { - val commandResult: CommandResult = %%("cwltool", "--quiet", "--print-pre", reference.pathAsString) - commandResult.exitCode match { - case 0 => commandResult.out.string - case error => - throw new RuntimeException( - s"running CwlTool on file ${reference.pathAsString} resulted in exit code $error and stderr ${commandResult.err.string}") - } - } -} - -/** - * Runs cwltool via heterodon. - * - * https://github.com/broadinstitute/heterodon - */ -final class CwltoolHeterodon extends CwltoolRunner { - // Trimmed down version of - // https://github.com/common-workflow-language/cwltool/blob/1a839255795882894b4bbfea6d909a74cacb1d6a/cwltool/main.py#L355 - private val cwltoolSaladExecScript = - """|import json - |import logging - | - |from cwltool.load_tool import fetch_document, resolve_tool_uri, validate_document - |from cwltool.loghandler import _logger - | - | - |def cwltool_salad(path): - | _logger.setLevel(logging.WARN) - | uri, tool_file_uri = resolve_tool_uri(path) - | document_loader, workflowobj, uri = fetch_document(uri) - | document_loader, avsc_names, processobj, metadata, uri \ - | = validate_document(document_loader, workflowobj, uri, preprocess_only=True) - | return json.dumps(processobj, indent=4) - |""".stripMargin - - private def cwltoolSaladEvalStatement(reference: CwlReference): String = s"cwltool_salad('${reference.pathAsString}')" - - def salad(reference: CwlReference): String = { - val execAndEval = new ExecAndEval() - execAndEval.apply(cwltoolSaladExecScript, cwltoolSaladEvalStatement(reference)).asInstanceOf[String] - } -} diff --git a/cwl/src/main/scala/cwl/EnumSchema.scala b/cwl/src/main/scala/cwl/EnumSchema.scala deleted file mode 100644 index 83ff1b05813..00000000000 --- a/cwl/src/main/scala/cwl/EnumSchema.scala +++ /dev/null @@ -1,35 +0,0 @@ -package cwl - -import cats.data.NonEmptyList -import shapeless.Witness -import wom.types.WomEnumerationType - -trait EnumSchema { - - val name: Option[String] - val symbols: Array[String] - val `type`: Witness.`"enum"`.T - val label: Option[String] - - def toWomEnumerationType: WomEnumerationType = { - val symbolIds = symbols.toList.map{ - s => s.substring(s.lastIndexOf("/") + 1) - } - WomEnumerationType(NonEmptyList.fromListUnsafe(symbolIds)) - } -} - -case class InputEnumSchema( - name: Option[String], - symbols: Array[String], - `type`: Witness.`"enum"`.T = Witness("enum").value, - label: Option[String] = None, - inputBinding: Option[InputCommandLineBinding] = None) extends EnumSchema - -case class OutputEnumSchema( - name: Option[String], - symbols: Array[String], - `type`: Witness.`"enum"`.T, - label: Option[String], - outputBinding: Option[CommandOutputBinding]) extends EnumSchema - diff --git a/cwl/src/main/scala/cwl/EvaluateExpression.scala b/cwl/src/main/scala/cwl/EvaluateExpression.scala deleted file mode 100644 index 00036d49de4..00000000000 --- a/cwl/src/main/scala/cwl/EvaluateExpression.scala +++ /dev/null @@ -1,20 +0,0 @@ -package cwl - -import common.validation.ErrorOr.ErrorOr -import cwl.ExpressionEvaluator.{ECMAScriptExpression, ECMAScriptFunction} -import shapeless.Poly1 -import wom.values.WomValue - -object EvaluateExpression extends Poly1 { - implicit val script: Case.Aux[ECMAScriptExpression, ParameterContext => ErrorOr[WomValue]] = { - at { - ExpressionEvaluator.evalExpression - } - } - - implicit val function: Case.Aux[ECMAScriptFunction, ParameterContext => ErrorOr[WomValue]] = { - at { - ExpressionEvaluator.evalFunction - } - } -} diff --git a/cwl/src/main/scala/cwl/ExpressionEvaluator.scala b/cwl/src/main/scala/cwl/ExpressionEvaluator.scala deleted file mode 100644 index a3ed2c7a7c4..00000000000 --- a/cwl/src/main/scala/cwl/ExpressionEvaluator.scala +++ /dev/null @@ -1,101 +0,0 @@ -package cwl - -import cats.syntax.validated._ -import common.validation.ErrorOr._ -import cwl.internal.{CwlEcmaScriptDecoder, EcmaScriptEncoder, EcmaScriptUtil} -import eu.timepit.refined.api.Refined -import eu.timepit.refined.string.MatchesRegex -import shapeless.Witness -import wom.callable.RuntimeEnvironment -import wom.values.{WomFloat, WomInteger, WomString, WomValue} - -// http://www.commonwl.org/v1.0/CommandLineTool.html#Expressions -object ExpressionEvaluator { - // A code fragment wrapped in the $(...) syntax must be evaluated as a ECMAScript expression. - /* - There are several places in cwltool where a simple contains check is used: - - https://github.com/common-workflow-language/cwltool/blob/353dbed/cwltool/builder.py#L157 - - https://github.com/common-workflow-language/cwltool/blob/353dbed/cwltool/command_line_tool.py#L669 - - https://github.com/common-workflow-language/cwltool/blob/353dbed/cwltool/expression.py#L219 - - https://github.com/common-workflow-language/cwltool/blob/353dbed/cwltool/process.py#L645 - */ - val ECMAScriptExpressionWitness = Witness("""(?s).*\$\(.*""") - val ECMAScriptExpressionRegex = ECMAScriptExpressionWitness.value.r - type MatchesECMAScriptExpression = MatchesRegex[ECMAScriptExpressionWitness.T] - type ECMAScriptExpression = String Refined MatchesECMAScriptExpression - - // A code fragment wrapped in the ${...} syntax must be evaluated as a ECMAScript function body for an anonymous, - // zero-argument function. - val ECMAScriptFunctionWitness = Witness("""(?s)\s*\$\{(.*)\}\s*""") - val ECMAScriptFunctionRegex = ECMAScriptFunctionWitness.value.r - type MatchesECMAScriptFunction = MatchesRegex[ECMAScriptFunctionWitness.T] - type ECMAScriptFunction = String Refined MatchesECMAScriptFunction - - def eval(expr: Expression, parameterContext: ParameterContext): ErrorOr[WomValue] = { - expr.fold(EvaluateExpression).apply(parameterContext) - } - - def evalExpression(expression: ECMAScriptExpression)(parameterContext: ParameterContext): ErrorOr[WomValue] = { - def evaluator(string: String): ErrorOr[WomValue] = { - eval(string, parameterContext) - } - - ExpressionInterpolator.interpolate(expression.value, evaluator, strip_whitespace = false) - } - - def evalFunction(function: ECMAScriptFunction)(parameterContext: ParameterContext): ErrorOr[WomValue] = { - function.value match { - case ECMAScriptFunctionRegex(script) => - - val functionExpression = - s"""|(function() { - |FUNCTION_BODY - |})(); - |""".stripMargin.replace("FUNCTION_BODY", script) - - eval(functionExpression, parameterContext) - case unmatched => - s"Expression '$unmatched' was unable to be matched to regex '${ECMAScriptFunctionWitness.value}'".invalidNel - } - } - - private lazy val cwlJsDecoder = new CwlEcmaScriptDecoder() - - def eval(expr: String, parameterContext: ParameterContext): ErrorOr[WomValue] = { - val script = if (parameterContext.expressionLib.isEmpty) { - expr - } else { - parameterContext.expressionLib.mkString("", ";", s";$expr") - } - val (rawVals, mapVals) = paramValues(parameterContext) - EcmaScriptUtil.evalStructish( - script, - rawVals, - mapVals, - new EcmaScriptEncoder(), - cwlJsDecoder - ) - } - - def paramValues(parameterContext: ParameterContext): ((String, WomValue), Map[String, Map[String, WomValue]]) = { - ( - "self" -> parameterContext.self - , - Map( - "inputs" -> parameterContext.inputs, - "runtime" -> parameterContext.runtimeOption.map(cwlMap).getOrElse(Map.empty) - ) - ) - } - - def cwlMap(runtime: RuntimeEnvironment): Map[String, WomValue] = { - Map( - "outdir" -> WomString(runtime.outputPath), - "tmpdir" -> WomString(runtime.tempPath), - "cores" -> WomInteger(runtime.cores.value), - "ram" -> WomFloat(runtime.ram), - "outdirSize" -> WomFloat(runtime.outputPathSize.toDouble), - "tmpdirSize" -> WomFloat(runtime.tempPathSize.toDouble) - ) - } -} diff --git a/cwl/src/main/scala/cwl/ExpressionInterpolator.scala b/cwl/src/main/scala/cwl/ExpressionInterpolator.scala deleted file mode 100644 index 9a40ee8e06f..00000000000 --- a/cwl/src/main/scala/cwl/ExpressionInterpolator.scala +++ /dev/null @@ -1,173 +0,0 @@ -package cwl - -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import common.validation.ErrorOr._ -import wom.types.WomNothingType -import wom.values._ - -import scala.jdk.CollectionConverters._ - -/** - * Partial copy-port of cwltool's expression.py. - * - * Minimizes refactor so that as expression.py continues to update the updates may be manually copied here also. - * - * Current omissions in `def interpolate`: - * - `def evaluator` - * - Utility not copy-ported. - * - `def interpolate` instead ignores absence of InlineJavascriptRequirement and instead always uses fullJS=True - * - `leaf = json.dumps(e, sort_keys=True)` - * - When the interpolated string is not just-a-single-expression there is undefined behavior of rendering - * non-WomPrimitive values into json. - * - Even if it did, does not sort the keys. - * - Thus we do not currently return character-for-character equality for these expressions with cwltool, however - * these cases are not explicitly defined in the CWL spec as far as I know. - * - * @see https://github.com/common-workflow-language/cwltool/blob/353dbed/cwltool/expression.py - */ -//noinspection ZeroIndexToHead,RemoveRedundantReturn -object ExpressionInterpolator { - - class SubstitutionException(message: String) extends RuntimeException(message) - - /** - * Copy-port of expression.py's scanner. - */ - private def scanner(scan: String): List[Int] = { - val DEFAULT = 0 - val DOLLAR = 1 - val PAREN = 2 - val BRACE = 3 - val SINGLE_QUOTE = 4 - val DOUBLE_QUOTE = 5 - val BACKSLASH = 6 - - var i = 0 - val stack = new java.util.Stack[Int] - stack.push(DEFAULT) - var start = 0 - while (i < scan.length) { - val state = stack.peek - val c = scan(i) - - if (state == DEFAULT) { - if (c == '$') { - stack.push(DOLLAR) - } else if (c == '\\') { - stack.push(BACKSLASH) - } - } else if (state == BACKSLASH) { - stack.pop() - if (stack.peek == DEFAULT) { - return List(i - 1, i + 1) - } - } else if (state == DOLLAR) { - if (c == '(') { - start = i - 1 - stack.push(PAREN) - } else if (c == '{') { - start = i - 1 - stack.push(BRACE) - } else { - stack.pop() - } - } else if (state == PAREN) { - if (c == '(') { - stack.push(PAREN) - } else if (c == ')') { - stack.pop() - if (stack.peek == DOLLAR) { - return List(start, i + 1) - } - } else if (c == '\'') { - stack.push(SINGLE_QUOTE) - } else if (c == '"') { - stack.push(DOUBLE_QUOTE) - } - } else if (state == BRACE) { - if (c == '{') { - stack.push(BRACE) - } else if (c == '}') { - stack.pop() - if (stack.peek == DOLLAR) { - return List(start, i + 1) - } - } else if (c == '\'') { - stack.push(SINGLE_QUOTE) - } else if (c == '"') { - stack.push(DOUBLE_QUOTE) - } - } else if (state == SINGLE_QUOTE) { - if (c == '\'') { - stack.pop() - } else if (c == '\\') { - stack.push(BACKSLASH) - } - } else if (state == DOUBLE_QUOTE) { - if (c == '"') { - stack.pop() - } else if (c == '\\') { - stack.push(BACKSLASH) - } - } - i += 1 - } - if (stack.size > 1) { - throw new SubstitutionException( - "Substitution error, unfinished block starting at position %s: %s".format(start, scan.drop(start))) - } else { - return null - } - } - - /** - * Copy-port of expression.py's interpolate. - */ - def interpolate(scanParam: String, - evaluator: String => ErrorOr[WomValue], - strip_whitespace: Boolean = true): ErrorOr[WomValue] = { - var scan = scanParam - if (strip_whitespace) { - scan = scan.trim - } - val parts = new java.util.Stack[ErrorOr[WomValue]] - var w = scanner(scan) - while (w != null) { - parts.push(WomString(scan.slice(0, w(0))).valid) - - if (scan(w(0)) == '$') { - val e = evaluator(scan.slice(w(0) + 1, w(1))) - if (w(0) == 0 && w(1) == scan.length && parts.size <= 1) { - return e - } - /* - Original-ish: - var leaf = json.dumps(e, sort_keys=True) - if (leaf(0) == '"') { - leaf = leaf.drop(1).dropRight(1) - } - Instead, push the raw WomValue. WomString's won't have been quoted as python's json.dumps does. - Other WomPrimitive's should be fine. CWL using other structures (Maps, Arrays, etc.) may have problems. - */ - val leaf = e - parts.push(leaf) - } else if (scan(w(0)) == '\\') { - val e = scan(w(1) - 1) - parts.push(WomString(e.toString).valid) - } - - scan = scan.drop(w(1)) - w = scanner(scan) - } - parts.push(WomString(scan).valid) - return parts.asScala.toList.sequence[ErrorOr, WomValue] map { list => - WomString(list.map{ - //we represent nulls as this type because Wom doesn't have a "null" value, but it does have a nothing type - case WomOptionalValue(WomNothingType, None) => "null" - case other => other.valueString - }.mkString("")) - } - } -} diff --git a/cwl/src/main/scala/cwl/ExpressionTool.scala b/cwl/src/main/scala/cwl/ExpressionTool.scala deleted file mode 100644 index 53c8a675a4c..00000000000 --- a/cwl/src/main/scala/cwl/ExpressionTool.scala +++ /dev/null @@ -1,136 +0,0 @@ -package cwl - -import cats.syntax.either._ -import cats.syntax.validated._ -import cats.instances.list._ -import common.Checked -import common.validation.Checked._ -import common.validation.ErrorOr.ErrorOr -import common.validation.Validation._ -import cwl.CwlVersion._ -import cwl.ExpressionTool.{ExpressionToolInputParameter, ExpressionToolOutputParameter} -import shapeless.{Coproduct, Witness} -import wom.RuntimeAttributes -import wom.callable.{Callable, CallableExpressionTaskDefinition} -import wom.expression.{IoFunctionSet, ValueAsAnExpression} -import wom.graph.GraphNodePort.OutputPort -import wom.types.{WomMapType, WomStringType, WomType} -import wom.values.{WomMap, WomObject, WomString, WomValue} - -case class ExpressionTool( - inputs: Array[ExpressionToolInputParameter] = Array.empty, - outputs: Array[ExpressionToolOutputParameter] = Array.empty, - `class`: Witness.`"ExpressionTool"`.T, - expression: StringOrExpression, - id: String, - requirements: Option[Array[Requirement]] = None, - hints: Option[Array[Hint]] = None, - label: Option[String] = None, - doc: Option[String] = None, - cwlVersion: Option[CwlVersion] = None, - `$namespaces`: Option[Map[String, String]] = None, - `$schemas`: Option[Array[String]] = None - ) extends Tool { - - def asCwl: Cwl = Coproduct[Cwl](this) - - def castWomValueAsMap(evaluatedExpression: WomValue): Checked[Map[String, WomValue]] = { - evaluatedExpression match { - case WomMap(WomMapType(WomStringType, _), map) => map.map{ - case (WomString(key), value) => key -> value - case (key, _) => throw new RuntimeException(s"saw a non-string value $key in wom map $evaluatedExpression typed with string keys ") - }.asRight - case obj: WomObject => obj.values.asRight - case _ => s"Could not cast value $evaluatedExpression to a Map[String, WomValue]".invalidNelCheck - } - } - - def buildTaskDefinition(taskName: String, - inputDefinitions: List[_ <: Callable.InputDefinition], - outputDefinitions: List[Callable.OutputDefinition], - runtimeAttributes: RuntimeAttributes, - requirementsAndHints: List[cwl.Requirement], - expressionLib: ExpressionLib): ErrorOr[CallableExpressionTaskDefinition] = { - - def evaluate(inputs: Map[String, WomValue], ioFunctionSet: IoFunctionSet, outputPorts: List[OutputPort]): Checked[Map[OutputPort, WomValue]] = { - val womExpression = expression match { - case StringOrExpression.String(str) => ValueAsAnExpression(WomString(str)) - case StringOrExpression.Expression(expr) => ECMAScriptWomExpression(expr, inputNames, expressionLib) - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - - // If we expect a certain type for - def coerce(womValue: WomValue, womType: WomType): Checked[WomValue] = womType.coerceRawValue(womValue).toChecked - - /* - * Look in the object (the result of the expression evaluation) for values to match output each port. - * Ideally we'd want to find a value for each output port declared by the tool. - * However since the spec is not clear on this and cwltool seems to ignore entirely the outputs declaration, - * we'll do the same for now, meaning we'll assign a value to the output port iff the result of the expression has one for them, - * otherwise we'll remove the port for the final map. - * If there are fields in the expression's result that do not have a matching declared output port, they won't be added either, - * because we'd have to create an output port for them now which would be useless anyway since nothing could point to it. - */ - def mapPortsToValues(values: Map[String, WomValue]): Checked[Map[OutputPort, WomValue]] = { - import cats.syntax.traverse._ - - val coercedValues: List[ErrorOr[(OutputPort, WomValue)]] = for { - outputPort <- outputPorts - value <- values.get(outputPort.internalName) - coerced = coerce(value, outputPort.womType).toValidated - } yield coerced.map(outputPort -> _) - - coercedValues.sequence[ErrorOr, (OutputPort, WomValue)].map(_.toMap).toEither - } - - for { - // Evaluate the expression - evaluatedExpression <- womExpression.evaluateValue(inputs, ioFunctionSet).toEither - /* - * We expect the result to be an object of the form: - * { - * "outputName": value, - * ... - * } - */ - womMap <- castWomValueAsMap(evaluatedExpression) - - mappedValues <- mapPortsToValues(womMap) - } yield mappedValues - } - - CallableExpressionTaskDefinition( - taskName, - evaluate, - runtimeAttributes, - Map.empty, - Map.empty, - outputDefinitions, - inputDefinitions, - // TODO: This doesn't work in all cases and it feels clunky anyway - find a way to sort that out - prefixSeparator = "#" - ).validNel - } -} - -object ExpressionTool { - - case class ExpressionToolInputParameter(id: String, - label: Option[String] = None, - secondaryFiles: Option[SecondaryFiles] = None, - format: Option[InputParameterFormat] = None, - streamable: Option[Boolean] = None, - doc: Option[Doc] = None, - inputBinding: Option[InputCommandLineBinding] = None, - default: Option[CwlAny] = None, - `type`: Option[MyriadInputType] = None) extends InputParameter - - case class ExpressionToolOutputParameter(id: String, - label: Option[String] = None, - secondaryFiles: Option[SecondaryFiles] = None, - format: Option[OutputParameterFormat] = None, - streamable: Option[Boolean] = None, - doc: Option[Doc] = None, - outputBinding: Option[CommandOutputBinding] = None, - `type`: Option[MyriadOutputType] = None) extends OutputParameter -} diff --git a/cwl/src/main/scala/cwl/FileParameter.scala b/cwl/src/main/scala/cwl/FileParameter.scala deleted file mode 100644 index 2413d9d7c02..00000000000 --- a/cwl/src/main/scala/cwl/FileParameter.scala +++ /dev/null @@ -1,150 +0,0 @@ -package cwl - -import cats.effect.IO -import cats.syntax.traverse._ -import cats.instances.list._ -import common.validation.ErrorOr._ -import common.validation.IOChecked._ -import common.validation.Validation._ -import cwl.ontology.Schema -import shapeless.Poly1 -import wom.expression.IoFunctionSet -import wom.types.{WomFileType, WomMaybePopulatedFileType} -import wom.values.{WomArray, WomFile, WomMaybePopulatedFile, WomValue} - -object FileParameter { - private val ReadLimit = Option(64 * 1024) - - def populateSecondaryFiles(womValue: WomValue, - secondaryFilesCoproduct: Option[SecondaryFiles], - formatOption: Option[String], - parameterContext: ParameterContext, - expressionLib: ExpressionLib, - ioFunctions: IoFunctionSet): IOChecked[WomValue] = { - - womValue match { - - case womMaybePopulatedFile: WomMaybePopulatedFile => - val secondaryFilesErrorOr = FileParameter.secondaryFiles( - womMaybePopulatedFile, - WomMaybePopulatedFileType, - secondaryFilesCoproduct, - parameterContext, - expressionLib, - ioFunctions - ) - - secondaryFilesErrorOr map { secondaryFiles => - womMaybePopulatedFile.copy(secondaryFiles = secondaryFiles, formatOption = formatOption) - } - - case womArray: WomArray => - womArray.value.toList.traverse( - populateSecondaryFiles(_, secondaryFilesCoproduct, formatOption, parameterContext, expressionLib, ioFunctions) - ).map(WomArray(_)) - - case womValue: WomValue => womValue.validIOChecked - } - } - - /** - * Checks if the file is compatible with a format. - */ - def checkFormat(womMaybePopulatedFile: WomMaybePopulatedFile, - formatsOption: Option[List[String]], - schemaOption: Option[Schema]): ErrorOr[Unit] = { - validate { - for { - schema <- schemaOption - fileFormat <- womMaybePopulatedFile.formatOption - formats <- formatsOption - } yield { - if (!formats.exists(schema.isSubClass(fileFormat, _))) - throw new RuntimeException(s"$fileFormat is not compatible with ${formats.mkString(", ")}") - } - () - } - } - - /** - * Populates the contents if they aren't loaded already. - */ - def maybeLoadContents(womMaybePopulatedFile: WomMaybePopulatedFile, - ioFunctionSet: IoFunctionSet, - loadContents: Boolean): IO[Option[String]] = { - womMaybePopulatedFile.contentsOption match { - case someContents@Some(_) => IO.pure(someContents) - case None if !loadContents => IO.pure(None) - case _ => FileParameter.load64KiB(womMaybePopulatedFile.value, ioFunctionSet).map(Option(_)) - } - } - - def load64KiB(path: String, ioFunctionSet: IoFunctionSet): IO[String] = { - implicit val ec = IO.contextShift(ioFunctionSet.ec) - IO.fromFuture(IO { ioFunctionSet.readFile(path, ReadLimit, failOnOverflow = false) }) - } - - /** - * Returns the list of secondary files for the primary file. - */ - def secondaryFiles(primaryWomFile: WomFile, - stringWomFileType: WomFileType, - secondaryFilesOption: Option[SecondaryFiles], - parameterContext: ParameterContext, - expressionLib: ExpressionLib, - ioFunctions: IoFunctionSet): IOChecked[List[WomFile]] = { - secondaryFilesOption - .map(secondaryFiles(primaryWomFile, stringWomFileType, _, parameterContext, expressionLib, ioFunctions)) - .getOrElse(List.empty[WomFile].validIOChecked) - } - - /** - * Returns the list of secondary files for the primary file. - */ - def secondaryFiles(primaryWomFile: WomFile, - stringWomFileType: WomFileType, - secondaryFiles: SecondaryFiles, - parameterContext: ParameterContext, - expressionLib: ExpressionLib, - ioFunctions: IoFunctionSet): IOChecked[List[WomFile]] = { - secondaryFiles - .fold(SecondaryFilesPoly) - .apply(primaryWomFile, stringWomFileType, parameterContext, expressionLib, ioFunctions) - } - - type SecondaryFilesFunction = (WomFile, WomFileType, ParameterContext, ExpressionLib, IoFunctionSet) => IOChecked[List[WomFile]] - - object SecondaryFilesPoly extends Poly1 { - implicit def caseStringOrExpression: Case.Aux[StringOrExpression, SecondaryFilesFunction] = { - at { - _.fold(this) - } - } - - implicit def caseExpression: Case.Aux[Expression, SecondaryFilesFunction] = { - at { - expression => - (primaryWomFile, stringWomFileType, parameterContext, expressionLib, ioFunctions) => - File.secondaryExpressionFiles(primaryWomFile, stringWomFileType, expression, parameterContext, expressionLib, ioFunctions).toIOChecked - } - } - - implicit def caseString: Case.Aux[String, SecondaryFilesFunction] = { - at { - string => - (primaryWomFile, stringWomFileType, _, _, ioFunctions) => - File.secondaryStringFile(primaryWomFile, stringWomFileType, string, ioFunctions).map(List(_)) - } - } - - implicit def caseArray: Case.Aux[Array[StringOrExpression], SecondaryFilesFunction] = { - at { - array => - (primaryWomFile, stringWomFileType, parameterContext, expressionLib, ioFunctions) => - val functions: List[SecondaryFilesFunction] = array.toList.map(_.fold(this)) - functions.flatTraverse(_ (primaryWomFile, stringWomFileType, parameterContext, expressionLib, ioFunctions)) - } - } - } - -} diff --git a/cwl/src/main/scala/cwl/FullyQualifiedName.scala b/cwl/src/main/scala/cwl/FullyQualifiedName.scala deleted file mode 100644 index faabfa44fc4..00000000000 --- a/cwl/src/main/scala/cwl/FullyQualifiedName.scala +++ /dev/null @@ -1,61 +0,0 @@ -package cwl - -import cwl.command.ParentName - -/** - * All of these classes decompose a "fully qualified" id into its constituent parts. They have unique types as - * they are seen in different parts of a CWL document and they differ in content. - * - * The fully qualified names are created by the Schema salad preprocessing step. - * - * @see Schema salad Identifier Resolution - */ -trait FullyQualifiedName { - def fileName: String - def id: String - def parent: Option[String] -} - -case class FileAndId private(fileName: String, parent: Option[String], id: String) extends FullyQualifiedName - -object FileAndId { - def apply(in: String)(implicit parent: ParentName): FileAndId = { - val Array(fileName, id) = in.split("#") - val cleanID = parent.stripParent(id) - FileAndId(fileName, parent.value, cleanID) - } -} - -case class FileStepAndId private(fileName: String, parent: Option[String], stepId: String, id: String) extends FullyQualifiedName - -object FileStepAndId { - def apply(in: String)(implicit parent: ParentName): FileStepAndId = { - val Array(fileName, id) = in.split("#") - val cleanID = parent.stripParent(id) - val Array(stepId, outputId) = cleanID.split("/") - FileStepAndId(fileName, parent.value, stepId, outputId) - } -} - -case class ArbitrarilyNested(fileName: String, parent: Option[String], id: String) extends FullyQualifiedName - -case class FileStepUUID(fileName: String, parent: Option[String], id: String, uuid: String, stepId: String) extends FullyQualifiedName - -object FullyQualifiedName { - def apply(in: String)(implicit parent: ParentName): FullyQualifiedName = maybeApply(in)(parent).getOrElse(throw new Exception(s"malformed FQN: $in")) - - def maybeApply(in: String)(implicit parent: ParentName): Option[FullyQualifiedName] = { - - in.split("#") match { - case Array(file, after) => - val cleanAfter = parent.stripParent(after) - cleanAfter.split("/").toList match { - case step :: uuid :: id :: Nil => Option(FileStepUUID(file, parent.value, id, uuid, step)) - case step :: id :: Nil => Option(FileStepAndId(file, parent.value, step, id)) - case id :: Nil => Option(FileAndId(file, parent.value, id)) - case many => Option(ArbitrarilyNested(file, parent.value, many.last)) - } - case _ => None - } - } -} diff --git a/cwl/src/main/scala/cwl/GlobEvaluator.scala b/cwl/src/main/scala/cwl/GlobEvaluator.scala deleted file mode 100644 index d50b154da2c..00000000000 --- a/cwl/src/main/scala/cwl/GlobEvaluator.scala +++ /dev/null @@ -1,67 +0,0 @@ -package cwl - -import cats.syntax.validated._ -import common.validation.ErrorOr._ -import shapeless._ -import wom.types.{WomArrayType, WomStringType} -import wom.values._ - -/* -CommandOutputBinding.glob: -Find files relative to the output directory, using POSIX glob(3) pathname matching. If an array is provided, find -files that match any pattern in the array. If an expression is provided, the expression must return a string or an -array of strings, which will then be evaluated as one or more glob patterns. Must only match and return files which -actually exist. - -http://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputBinding - */ -object GlobEvaluator { - - type GlobEvaluatorFunction = (ParameterContext, ExpressionLib) => ErrorOr[List[String]] - - def globs(globOption: Option[Glob], parameterContext: ParameterContext, expressionLib: ExpressionLib): ErrorOr[List[String]] = { - globOption map { - globs(_, parameterContext, expressionLib) - } getOrElse { - Nil.valid - } - } - - def globs(glob: Glob, parameterContext: ParameterContext, expressionLib: ExpressionLib): ErrorOr[List[String]] = { - glob.fold(GlobEvaluator.GlobEvaluatorPoly).apply(parameterContext, expressionLib) - } - - object GlobEvaluatorPoly extends Poly1 { - implicit def caseStringOrExpression: Case.Aux[StringOrExpression, GlobEvaluatorFunction] = { - at { - _.fold(this) - } - } - - implicit def caseExpression: Case.Aux[Expression, GlobEvaluatorFunction] = { - at { - expression => - (parameterContext, expressionLib) => { - ExpressionEvaluator.eval(expression, parameterContext) flatMap { - case WomArray(_, values) if values.isEmpty => Nil.valid - case WomString(value) => List(value).valid - case WomArray(WomArrayType(WomStringType), values) => values.toList.map(_.valueString).valid - case womValue => - val message = s"Unexpected expression result: $womValue while evaluating expression '$expression' " + - s"using inputs '$parameterContext'" - message.invalidNel - } - } - } - } - - implicit def caseArrayString: Case.Aux[Array[String], GlobEvaluatorFunction] = { - at { array => (_, _) => array.toList.valid } - } - - implicit def caseString: Case.Aux[String, GlobEvaluatorFunction] = { - at { string => (_, _) => List(string).valid } - } - } - -} diff --git a/cwl/src/main/scala/cwl/InitialWorkDirRequirement.scala b/cwl/src/main/scala/cwl/InitialWorkDirRequirement.scala deleted file mode 100644 index 28e49fe71d9..00000000000 --- a/cwl/src/main/scala/cwl/InitialWorkDirRequirement.scala +++ /dev/null @@ -1,63 +0,0 @@ -package cwl - -import cwl.InitialWorkDirRequirement._ -import eu.timepit.refined.W -import shapeless.{:+:, CNil, _} - -final case class InitialWorkDirRequirement( - `class`: W.`"InitialWorkDirRequirement"`.T, - listing: IwdrListing - ) { - val listings: Array[IwdrListingArrayEntry] = listing.fold(IwdrListingArrayPoly) - - override def toString: WorkflowStepInputId = - s"""InitialWorkDirRequirement( - | ${listings.mkString(System.lineSeparator + " ")} - |)""".stripMargin -} - -/** - * Short for "Directory Entry" - * @see Dirent Specification - * - * Split into two cases because entryName is only optional if entry is an Expression - */ -trait Dirent { - def writable: Option[Boolean] - def writableWithDefault = writable.getOrElse(false) -} - -final case class ExpressionDirent( - entry: Expression, - entryname: Option[StringOrExpression], - writable: Option[Boolean] - ) extends Dirent - -final case class StringDirent( - entry: String, - entryname: StringOrExpression, - writable: Option[Boolean] - ) extends Dirent - -object InitialWorkDirRequirement { - - // "ExpressionDirent" has to come before StringDirent because expressions are matched by "String" first if we do it the - // other way round. - final type IwdrListingArrayEntry = File :+: Directory :+: ExpressionDirent :+: StringDirent :+: StringOrExpression :+: CNil - final type IwdrListing = Array[IwdrListingArrayEntry] :+: StringOrExpression :+: CNil - - object IwdrListingArrayPoly extends Poly1 { - implicit val caseArrayIwdrListingArrayEntry: Case.Aux[Array[IwdrListingArrayEntry], Array[IwdrListingArrayEntry]] = { - at { - identity - } - } - - implicit val caseStringOrExpression: Case.Aux[StringOrExpression, Array[IwdrListingArrayEntry]] = { - at { - stringOrExpression => - Array(Coproduct[IwdrListingArrayEntry](stringOrExpression)) - } - } - } -} diff --git a/cwl/src/main/scala/cwl/InputParameter.scala b/cwl/src/main/scala/cwl/InputParameter.scala deleted file mode 100644 index d8b1579dcef..00000000000 --- a/cwl/src/main/scala/cwl/InputParameter.scala +++ /dev/null @@ -1,190 +0,0 @@ -package cwl - -import cats.syntax.functor._ -import cats.syntax.parallel._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import cats.instances.option._ -import common.validation.ErrorOr._ -import common.validation.IOChecked._ -import common.validation.Validation._ -import cwl.FileParameter._ -import cwl.ontology.Schema -import shapeless.Poly1 -import wom.callable.Callable.InputDefinition.InputValueMapper -import wom.expression.IoFunctionSet -import wom.types.{WomSingleFileType, WomType} -import wom.values.{WomArray, WomMaybeListedDirectory, WomMaybePopulatedFile, WomObject, WomObjectLike, WomOptionalValue, WomValue} - -trait InputParameter { - def id: String - def label: Option[String] - def secondaryFiles: Option[SecondaryFiles] - def format: Option[InputParameterFormat] - def streamable: Option[Boolean] - def doc: Option[Doc] - def inputBinding: Option[InputCommandLineBinding] - def default: Option[CwlAny] - def `type`: Option[MyriadInputType] - def loadContents = inputBinding.flatMap(_.loadContents).getOrElse(false) -} - -object InputParameter { - object IdDefaultAndType { - def unapply(arg: InputParameter): Option[(String, CwlAny, MyriadInputType)] = (arg.default, arg.`type`) match { - case (Some(default), Some(tpe)) => Option((arg.id, default, tpe)) - case _ => None - } - } - - object IdAndType { - def unapply(arg: InputParameter): Option[(String, MyriadInputType)] = (arg.default, arg.`type`) match { - case (None, Some(tpe)) => Option((arg.id, tpe)) - case _ => None - } - } - - type DefaultToWomValueFunction = WomType => ErrorOr[WomValue] - - object DefaultToWomValuePoly extends Poly1 { - implicit def caseFileOrDirectory: Case.Aux[FileOrDirectory, DefaultToWomValueFunction] = { - at { - _.fold(this) - } - } - - implicit def caseFileOrDirectoryArray: Case.Aux[Array[FileOrDirectory], DefaultToWomValueFunction] = { - at { - fileOrDirectoryArray => - womType => - fileOrDirectoryArray - .toList - .traverse(_.fold(this).apply(womType)) - .map(WomArray(_)) - } - } - - implicit def caseFile: Case.Aux[File, DefaultToWomValueFunction] = { - at { - file => - womType => - file.asWomValue.flatMap(womType.coerceRawValue(_).toErrorOr) - } - } - - implicit def caseDirectory: Case.Aux[Directory, DefaultToWomValueFunction] = { - at { - directory => - womType => - directory.asWomValue.flatMap(womType.coerceRawValue(_).toErrorOr) - } - } - - implicit def caseJson: Case.Aux[io.circe.Json, DefaultToWomValueFunction] = { - at { - circeJson => - womType => - val stringJson = circeJson.noSpaces - import spray.json._ - val sprayJson = stringJson.parseJson - womType.coerceRawValue(sprayJson).toErrorOr - } - } - } - - object InputParameterFormatPoly extends Poly1 { - implicit val caseExpression: Case.Aux[Expression, ParameterContext => ErrorOr[List[String]]] = { - at { expression => - parameterContext => - ExpressionEvaluator.eval(expression, parameterContext) map { - case WomArray(_, values) => values.toList.map(_.valueString) - case womValue => List(womValue.valueString) - } - } - } - - implicit val caseString: Case.Aux[String, ParameterContext => ErrorOr[List[String]]] = { - at { string => - _ => - List(string).valid - } - } - - implicit val caseArrayString: Case.Aux[Array[String], ParameterContext => ErrorOr[List[String]]] = { - at { array => - _ => - array.toList.valid - } - } - } - - /** - * Yet another value mapper. This one is needed because in CWL we might need to "augment" inputs which we can only do - * once they have been linked to a WomValue. This input value mapper encapsulates logic to be applied once that is - * done. For now, if the inputParameter has an input binding with loadContents = true, load the content of the file. - * - * This is based on the spec in http://www.commonwl.org/v1.0/CommandLineTool.html#Input_binding - * - * NOTE: There may be _many_ cases not implemented here that need to be fixed. - */ - def inputValueMapper(inputParameter: InputParameter, - inputType: MyriadInputType, - expressionLib: ExpressionLib, - schemaOption: Option[Schema]): InputValueMapper = { - ioFunctionSet: IoFunctionSet => { - import ioFunctionSet.cs - - def populateFiles(womValue: WomValue): IOChecked[WomValue] = { - womValue match { - case womMaybePopulatedFile: WomMaybePopulatedFile => - // Don't include the secondary files in the self variables - val parameterContext = ParameterContext(ioFunctionSet, expressionLib, self = womMaybePopulatedFile.copy(secondaryFiles = List.empty)) - val secondaryFilesFromInputParameter = inputParameter.secondaryFiles - val secondaryFilesFromType = inputType.fold(MyriadInputTypeToSecondaryFiles) - val secondaryFiles = secondaryFilesFromInputParameter orElse secondaryFilesFromType - val inputFormatsErrorOr = inputParameter.format - .traverse(_.fold(InputParameterFormatPoly).apply(parameterContext)) - - for { - inputFormatsOption <- inputFormatsErrorOr.toIOChecked - _ <- checkFormat(womMaybePopulatedFile, inputFormatsOption, schemaOption).toIOChecked - contentsOption <- FileParameter.maybeLoadContents( - womMaybePopulatedFile, - ioFunctionSet, - inputParameter.loadContents - ).to[IOChecked] - withSize <- womMaybePopulatedFile.withSize(ioFunctionSet).to[IOChecked] - loaded = withSize.copy(contentsOption = contentsOption) - secondaries <- FileParameter.secondaryFiles( - loaded, - WomSingleFileType, - secondaryFiles, - parameterContext, - expressionLib, - ioFunctionSet - ) - updated = loaded.copy(secondaryFiles = loaded.secondaryFiles ++ secondaries) - } yield updated - case womMaybeListedDirectory: WomMaybeListedDirectory => womMaybeListedDirectory.withSize(ioFunctionSet).to[IOChecked].widen - case WomArray(_, values) => values.toList.parTraverse[IOChecked, WomValue](populateFiles).map(WomArray(_)) - case WomOptionalValue(_, Some(innerValue)) => populateFiles(innerValue).map(WomOptionalValue(_)) - case obj: WomObjectLike => - // Map the values - val populated: IOChecked[WomObject] = obj.values.toList.parTraverse[IOChecked, (String, WomValue)]({ - case (key, value) => populateFiles(value).map(key -> _) - }) - .map(_.toMap) - // Validate new types are still valid w.r.t the object - .flatMap(WomObject.withTypeChecked(_, obj.womObjectTypeLike).toIOChecked) - - populated.widen[WomValue] - case womValue: WomValue => - pure(womValue) - } - } - - womValue => populateFiles(womValue) - } - } -} diff --git a/cwl/src/main/scala/cwl/LinkMergeMethod.scala b/cwl/src/main/scala/cwl/LinkMergeMethod.scala deleted file mode 100644 index bd54ca6b56a..00000000000 --- a/cwl/src/main/scala/cwl/LinkMergeMethod.scala +++ /dev/null @@ -1,8 +0,0 @@ -package cwl - -object LinkMergeMethod extends Enumeration { - type LinkMergeMethod = Value - - val MergeNested = Value("merge_nested") - val MergeFlattened = Value("merge_flattened") -} diff --git a/cwl/src/main/scala/cwl/MyriadInputTypeToSecondaryFiles.scala b/cwl/src/main/scala/cwl/MyriadInputTypeToSecondaryFiles.scala deleted file mode 100644 index 4ddc09625d3..00000000000 --- a/cwl/src/main/scala/cwl/MyriadInputTypeToSecondaryFiles.scala +++ /dev/null @@ -1,25 +0,0 @@ -package cwl - -import cwl.CwlType.CwlType -import shapeless.Poly1 - -// IAS.secondaryFiles are NOT listed in 1.0 spec, but according to jgentry they will be, maybe -object MyriadInputTypeToSecondaryFiles extends Poly1 { - implicit val caseMyriadInputInnerType: Case.Aux[MyriadInputInnerType, Option[SecondaryFiles]] = at { - _.fold(MyriadInputInnerTypeToSecondaryFiles) - } - - implicit val caseArrayMyriadInputInnerType: Case.Aux[Array[MyriadInputInnerType], Option[SecondaryFiles]] = at { - _.to(LazyList).flatMap(_.fold(MyriadInputInnerTypeToSecondaryFiles)).headOption - } -} - -object MyriadInputInnerTypeToSecondaryFiles extends Poly1 { - implicit val caseCwlType: Case.Aux[CwlType, Option[SecondaryFiles]] = at { _ => None } - implicit val caseInputRecordSchema: Case.Aux[InputRecordSchema, Option[SecondaryFiles]] = at { _ => None } - implicit val caseInputEnumSchema: Case.Aux[InputEnumSchema, Option[SecondaryFiles]] = at { _ => None } - implicit val caseInputArraySchema: Case.Aux[InputArraySchema, Option[SecondaryFiles]] = at { - inputArraySchema => inputArraySchema.secondaryFiles - } - implicit val caseString: Case.Aux[String, Option[SecondaryFiles]] = at { _ => None } -} diff --git a/cwl/src/main/scala/cwl/MyriadInputTypeToSortedCommandParts.scala b/cwl/src/main/scala/cwl/MyriadInputTypeToSortedCommandParts.scala deleted file mode 100644 index 498e97c016f..00000000000 --- a/cwl/src/main/scala/cwl/MyriadInputTypeToSortedCommandParts.scala +++ /dev/null @@ -1,254 +0,0 @@ -package cwl - -import cwl.CommandLineTool.{CommandBindingSortingKey, CommandPartsList, SortKeyAndCommandPart, StringOrInt} -import cwl.CwlType.CwlType -import cwl.SchemaDefRequirement.SchemaDefTypes -import cwl.command.ParentName -import shapeless.{Coproduct, Poly1} -import wom.values.{WomArray, WomCoproductValue, WomObjectLike, WomOptionalValue, WomValue} -import cats.syntax.option._ -import cats.syntax.traverse._ -import cats.instances.list._ -import cats.instances.option._ - -/** - * Poly1 to fold over a MyriadInputType and create a List of SortingKeyAndCommandPart based on the rules described here: - * http://www.commonwl.org/v1.0/CommandLineTool.html#Input_binding - */ -object MyriadInputTypeToSortedCommandParts extends Poly1 { - import Case._ - - /** - * CommandLineBinding: binding of the element being processed. - * It's possible for an input not to have an "inputBinding", but to have a complex type which itself has inputBindings for its fields or items. - * e.g: - * - id: arrayInput - * type: - * type: array - * items: File - * inputBinding: { prefix: "-YYY" } - * - * Here the array type has an inputBinding but the input itself (arrayInput) does not. - * This would yield on the command line something like "-YYY arrayItem0 -YYY arrayItem1 ..." - * - * On the other hand: - * - id: arrayInput - * type: - * type: array - * items: File - * inputBinding: { prefix: "-YYY" } - * inputBinding: { prefix: "-XXX" } - * - * In this case the input itself has a binding as well, which would yield something like "-XXX -YYY arrayItem0 -YYY arrayItem1 ..." - * - * For arrays specifically, the presence of an "itemSeparator" and / or "valueFrom" in the input binding will change the way the array is processed, - * which is why we need to know about it in this Poly1 that folds over the input types. - * - * WomValue: value bound to the element (in the cases above it would be the WomArray) - * CommandBindingSortingKey: The current sorting key. Because we might need to recurse in nested types we need to propagate the key as we do. - */ - type CommandPartBuilder = (Option[InputCommandLineBinding], WomValue, CommandBindingSortingKey, Boolean, ExpressionLib, SchemaDefRequirement) => CommandPartsList - - implicit def m: Aux[MyriadInputInnerType, CommandPartBuilder] = { - at { - innerType => { - (binding, womValue, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) => - innerType - .fold(MyriadInputInnerTypeToSortedCommandParts) - .apply(binding, womValue, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement).fold( - throw new RuntimeException(s"inner type to command part failed! on type $innerType w/ womValue $womValue"))(identity) - } - } - } - - implicit def am: Aux[Array[MyriadInputInnerType], CommandPartBuilder] = { - at { - types => { - (binding, womValue, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) => - def lookupTypes(innerTypes: Array[MyriadInputInnerType]) = { - innerTypes.toList.map{ - _.fold(MyriadInputInnerTypeToSortedCommandParts) - .apply(binding, womValue, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) - }.reduce(_ orElse _) - } - val result: Option[CommandPartsList] = - types.partition(_.select[CwlType].contains(CwlType.Null)) match { - case (Array(_), types) => lookupTypes(types) orElse Some(List.empty[SortKeyAndCommandPart]) - case (Array(), types) => lookupTypes(types) - } - result.fold( - throw new RuntimeException(s"could not produce command line parts from input $womValue and types $types"))( - identity - ) - } - } - } -} - -object MyriadInputInnerTypeToSortedCommandParts extends Poly1 { - - type CommandPartBuilder = (Option[InputCommandLineBinding], WomValue, CommandBindingSortingKey, Boolean, ExpressionLib, SchemaDefRequirement) => Option[CommandPartsList] - - import Case._ - - // Primitive type: we just need to create a command part from the binding if there's one here. - implicit def ct: Aux[CwlType, CommandPartBuilder] = { - at { - cwlType => { - case (_, WomOptionalValue(_, None), _, _, _, _) => List.empty.some - case (inputBinding, womValue, key, hasShellCommandRequirement, expressionLib, _) => { - (cwlType, womValue) match { - case (CwlType.Null, _) => List.empty.some - case (_, WomOptionalValue(_, None)) => List.empty.some - case (_,_) => inputBinding.toList.map(_.toCommandPart(key, womValue, hasShellCommandRequirement, expressionLib)).some - } - } - } - } - } - - implicit def irs: Aux[InputRecordSchema, CommandPartBuilder] = at[InputRecordSchema] { irs => { - def go: CommandPartBuilder = { - - //If the value is optional and is supplied, recurse over the value provided - case (inputBinding, WomCoproductValue(_, value), sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) => - go(inputBinding, value, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) - - //If the value is optional and is supplied, recurse over the value provided - case (inputBinding, WomOptionalValue(_, Some(value)), sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) => - go(inputBinding, value, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) - - // If it's optional and there's no value, do nothing - case (_, WomOptionalValue(_, None), _, _, _, _) => List.empty.some - - // If there's no input binding and no input bindings within the irs, do nothing - case (None, _, _, _, _, _) if !irs.fields.exists(_.exists(_.inputBinding.isDefined)) => List.empty.some - - case (inputBinding, objectLike: WomObjectLike, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) => - // If there's an input binding, make a SortKeyAndCommandPart for it - val sortingKeyFromInputBindingFromInputParameter: Option[SortKeyAndCommandPart] = - inputBinding.map(_.toCommandPart(sortingKey, objectLike, hasShellCommandRequirement, expressionLib)) - - // iterate through the fields and fold over their type - val partsFromFields:Option[CommandPartsList] = - irs.fields.toList.flatten. - flatTraverse{ - case InputRecordField(name, tpe, _, inputBinding, _) => - // Parse the name to get a clean id - val parsedName = FullyQualifiedName(name)(ParentName.empty).id - - // The field name needs to be added to the key after the input binding (as per the spec) - // Also start from the key from the input binding if there was one - val fieldSortingKey = - sortingKeyFromInputBindingFromInputParameter. - map(_.sortingKey). - getOrElse(sortingKey). - append(inputBinding, Coproduct[StringOrInt](parsedName)) - - val innerValueOption: Option[WomValue] = objectLike.values.get(parsedName) - - val folded = innerValueOption.map(tpe.fold(MyriadInputTypeToSortedCommandParts). - apply(inputBinding, _, fieldSortingKey.asNewKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement)) - folded - } - - sortingKeyFromInputBindingFromInputParameter.fold( - partsFromFields )( - sortingKeyFromInputBindingFromInputParameter => - partsFromFields.map{k => List(sortingKeyFromInputBindingFromInputParameter.copy(nestedCommandParts = k)) } - ) - case (_, other, _, _, _, _) => throw new RuntimeException(s"Value $other cannot be used for an input of type InputRecordSchema") - } - - go - }} - - implicit def ies: Aux[InputEnumSchema, CommandPartBuilder] = at[InputEnumSchema] { inputEnumSchema => - def go: CommandPartBuilder = { - case (_, WomOptionalValue(_, None), _, _, _, _) => List.empty.some - case (inputBinding, value, sortingKey, hasShellCommandRequirement, expressionLib, _) => - // If there's an input binding, make a SortKeyAndCommandPart for it - val fromInputBinding = - inputBinding.map(_.toCommandPart(sortingKey, value, hasShellCommandRequirement, expressionLib)).toList - - val fromIes = inputEnumSchema.inputBinding.map(_.toCommandPart(sortingKey, value, hasShellCommandRequirement, expressionLib)).toList - - (fromInputBinding ++ fromIes).some - } - go - } - - implicit def ias: Aux[InputArraySchema, CommandPartBuilder] = at[InputArraySchema] { ias => - def go: CommandPartBuilder = { - case (inputBindingFromInputParameterParent, WomOptionalValue(_, Some(value)), sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) => - go(inputBindingFromInputParameterParent, value, sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) - - //If it's optional and there's no value, do nothing - case (_, WomOptionalValue(_, None), _, _, _, _) => List.empty.some - - // If there's no input binding and no input bindings for the ias, do nothing - case (None, _, _, _, _, _) if ias.inputBinding.isEmpty => List.empty.some - - case (inputBinding, WomArray.WomArrayLike(womArray: WomArray), sortingKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement) => - - - // If there's an input binding, make a SortKeyAndCommandPart for it - val sortKeyFromInputBindingFromInputerParameterParent: Option[SortKeyAndCommandPart] = - inputBinding. - map(_.toCommandPart(sortingKey, womArray, hasShellCommandRequirement, expressionLib)) - - // Now depending on whether we have an itemSeparator and/or valueFrom or not, we're going to recurse over each element of the array (or not). - // See http://www.commonwl.org/v1.0/CommandLineTool.html#CommandLineBinding - if (inputBinding.flatMap(_.itemSeparator).isDefined || inputBinding.flatMap(_.valueFrom).isDefined) { - // If there's an item separator or a valueFrom we can stop here. - // When the command part is instantiated (see CommandLineBindingCommandPart) it will evaluate the valueFrom (if defined) and join the items together (if there's an itemSeparator). - sortKeyFromInputBindingFromInputerParameterParent.toList.some - } else { - // If neither valueFrom nor itemSeparator were defined, we need to process each item of the array - val fromArray: Option[CommandPartsList] = womArray.value.zipWithIndex.toList.flatTraverse({ - case (item, index) => - // Update the sorting key with the binding position (if any), add the index - val itemSortingKey = { - - // The index needs to be added to the key after the input binding (as per the spec) - // Also start from the key from the input binding if there was one - val sortKey = sortKeyFromInputBindingFromInputerParameterParent.fold(sortingKey)(_.sortingKey) - - sortKey.append(ias.inputBinding, Coproduct[StringOrInt](index)) - } - - // Even if the item doesn't have an explicit input binding, it should appear in the command so create a default empty one - //there is an explicit input binding! - val arrayItemInputBinding = - ias. - inputBinding. - orElse(Option(InputCommandLineBinding.default)) - - // Fold over the item type of each array element - Option(ias.items.fold(MyriadInputTypeToSortedCommandParts).apply(arrayItemInputBinding, item, itemSortingKey.asNewKey, hasShellCommandRequirement, expressionLib, schemaDefRequirement)) - }) - sortKeyFromInputBindingFromInputerParameterParent.fold(fromArray){ - sortKeyFromInputBindingFromInputerParameterParent => - fromArray.map{k => List(sortKeyFromInputBindingFromInputerParameterParent.copy(nestedCommandParts = k)) } - } - } - - case (_, other, _, _, _, _) => throw new RuntimeException(s"Value $other cannot be used for an input of type InputArraySchema") - } - - go - } - - implicit def s: Aux[String, CommandPartBuilder] = { - at { - s => { - case (_, WomOptionalValue(_, None), _, _, _, _) => List.empty.some - case (commandLineBindingFromInput, womValue, sortingKey, boolean, expressionLib, schemaDefRequirement) => { - val womType: SchemaDefTypes = schemaDefRequirement.lookupCwlType(s).getOrElse(throw new RuntimeException(s"Looked for type $s in custom types $schemaDefRequirement but no match was found!")) - - womType.fold(this).apply(commandLineBindingFromInput, womValue, sortingKey, boolean, expressionLib, schemaDefRequirement) - } - } - } - } -} diff --git a/cwl/src/main/scala/cwl/MyriadInputTypeToWomType.scala b/cwl/src/main/scala/cwl/MyriadInputTypeToWomType.scala deleted file mode 100644 index d3be698bd33..00000000000 --- a/cwl/src/main/scala/cwl/MyriadInputTypeToWomType.scala +++ /dev/null @@ -1,95 +0,0 @@ -package cwl - -import cats.data.NonEmptyList -import cwl.CwlType.CwlType -import cwl.MyriadInputTypeToWomType.SchemaLookup -import cwl.command.ParentName -import mouse.all._ -import shapeless.Poly1 -import wom.types._ - -object MyriadInputInnerTypeToString extends Poly1 { - implicit def ct = at[CwlType]{ _.toString } - implicit def irs = at[InputRecordSchema]{_.toString} - implicit def ies = at[InputEnumSchema]{ _.toString } - implicit def ias = at[InputArraySchema]{ _.toString} - implicit def s = at[String]{identity} -} - -object MyriadInputTypeToWomType extends Poly1 { - - import Case._ - - type SchemaLookup = SchemaDefRequirement => WomType - - implicit def m:Aux[MyriadInputInnerType, SchemaLookup]= at[MyriadInputInnerType] {_.fold(MyriadInputInnerTypeToWomType)} - - // An array of type means "this input value can be in any of those types." - // Currently we only accept single types or [null, X] to mean Optional[X] - implicit def am: Aux[Array[MyriadInputInnerType], SchemaLookup] = at[Array[MyriadInputInnerType]] { - types => - schemaLookup => - types.partition(_.select[CwlType].contains(CwlType.Null)) match { - // If there's a single non null type, use that - case (Array(), Array(singleNonNullType)) => - singleNonNullType.fold(MyriadInputInnerTypeToWomType).apply(schemaLookup) - case (Array(), array: Array[MyriadInputInnerType]) if array.length > 1 => - val types = array.map(_.fold(MyriadInputInnerTypeToWomType).apply(schemaLookup)) - WomCoproductType(NonEmptyList.fromListUnsafe(types.toList)) - // If there's a null type and a single non null type, it's a WomOptionalType - case (Array(_), Array(singleNonNullType)) => - WomOptionalType(singleNonNullType.fold(MyriadInputInnerTypeToWomType).apply(schemaLookup)) - case (Array(_), array: Array[MyriadInputInnerType]) if array.length > 1 => - val types = array.map(_.fold(MyriadInputInnerTypeToWomType).apply(schemaLookup)) - WomOptionalType(WomCoproductType(NonEmptyList.fromListUnsafe(types.toList))) - case (Array(_), _) => - val readableTypes = types.map(_.fold(MyriadInputInnerTypeToString)).mkString(", ") - throw new UnsupportedOperationException(s"Cromwell only supports single types or optionals (as indicated by [null, X]). Instead we saw: $readableTypes") - } - } -} - -object MyriadInputInnerTypeToWomType extends Poly1 { - import Case._ - - def ex(component: String) = throw new RuntimeException(s"input type $component not yet suported by WOM!") - - implicit def ct: Aux[CwlType, SchemaLookup] = at[CwlType]{ - ct => - cwl.cwlTypeToWomType(ct) |> Function.const - } - - def inputRecordSchemaToWomType(irs: InputRecordSchema): SchemaLookup = { schemaLookup: SchemaDefRequirement => - irs match { - case InputRecordSchema(_, Some(fields), _, _) => - val typeMap = fields.map({ field => - FullyQualifiedName(field.name)(ParentName.empty).id -> field.`type`.fold(MyriadInputTypeToWomType).apply(schemaLookup) - }).toMap - WomCompositeType(typeMap) - case irs => irs.toString |> ex - } - } - - implicit def irs: Aux[InputRecordSchema, SchemaLookup] = at[InputRecordSchema]{ - inputRecordSchemaToWomType - } - - implicit def ies: Aux[InputEnumSchema, SchemaLookup] = at[InputEnumSchema]{ - ies => - ies.toWomEnumerationType |> Function.const - } - - implicit def ias: Aux[InputArraySchema, SchemaLookup] = at[InputArraySchema]{ - ias => - lookup => - val arrayType: WomType = ias.items.fold(MyriadInputTypeToWomType).apply(lookup) - WomArrayType(arrayType) - } - - implicit def s: Aux[String, SchemaLookup] = at[String]{ - string => - schemaReq => - schemaReq.lookupType(string).getOrElse(throw new RuntimeException(s"Custom type $string was referred to but not found in schema def ${schemaReq}.")) - } - -} diff --git a/cwl/src/main/scala/cwl/MyriadOutputTypeToWomFiles.scala b/cwl/src/main/scala/cwl/MyriadOutputTypeToWomFiles.scala deleted file mode 100644 index 7c007d07d46..00000000000 --- a/cwl/src/main/scala/cwl/MyriadOutputTypeToWomFiles.scala +++ /dev/null @@ -1,66 +0,0 @@ -package cwl - -import cats.syntax.traverse._ -import cats.instances.list._ -import common.validation.IOChecked._ -import cwl.CwlType.CwlType -import cwl.MyriadOutputTypeToWomFiles.EvaluationFunction -import mouse.all._ -import shapeless.Poly1 -import wom.expression.FileEvaluation - -object MyriadOutputTypeToWomFiles extends Poly1 { - - type EvaluationFunction = CommandOutputBinding => IOChecked[Set[FileEvaluation]] - - import Case._ - - implicit def cwlType: Aux[MyriadOutputInnerType, EvaluationFunction => IOChecked[Set[FileEvaluation]]] = at[MyriadOutputInnerType]{ - _.fold(MyriadOutputInnerTypeToWomFiles) - } - - implicit def acwl: Aux[Array[MyriadOutputInnerType], EvaluationFunction => IOChecked[Set[FileEvaluation]]] = at[Array[MyriadOutputInnerType]] { types => - evalFunction => - types.toList.traverse(_.fold(MyriadOutputInnerTypeToWomFiles).apply(evalFunction)).map(_.toSet.flatten) - } -} - -object MyriadOutputInnerTypeToWomFiles extends Poly1 { - - import Case._ - - def ex(component: String) = throw new RuntimeException(s"output type $component cannot yield wom files") - - implicit def cwlType: Aux[CwlType, EvaluationFunction => IOChecked[Set[FileEvaluation]]] = at[CwlType] { _ =>_ => - Set.empty[FileEvaluation].validIOChecked - } - - implicit def ors: Aux[OutputRecordSchema, EvaluationFunction => IOChecked[Set[FileEvaluation]]] = at[OutputRecordSchema] { - case OutputRecordSchema(_, Some(fields), _) => - evalFunction => - fields.toList.traverse[IOChecked, Set[FileEvaluation]]({ field => - field.outputBinding match { - case Some(binding) => evalFunction(binding) - case None => field.`type`.fold(MyriadOutputTypeToWomFiles).apply(evalFunction) - } - }).map(_.toSet.flatten) - case ors => ors.toString |> ex - } - - implicit def oes: Aux[OutputEnumSchema, EvaluationFunction => IOChecked[Set[FileEvaluation]]] = at[OutputEnumSchema]{ oes =>_ => - oes.toString |> ex - } - - implicit def oas: Aux[OutputArraySchema, EvaluationFunction => IOChecked[Set[FileEvaluation]]] = at[OutputArraySchema]{ oas => - evalFunction => - import cats.syntax.apply._ - def fromBinding: IOChecked[Set[FileEvaluation]] = oas.outputBinding.map(evalFunction).getOrElse(Set.empty[FileEvaluation].validIOChecked) - def fromType: IOChecked[Set[FileEvaluation]] = oas.items.fold(MyriadOutputTypeToWomFiles).apply(evalFunction) - - (fromBinding, fromType) mapN (_ ++ _) - } - - implicit def s: Aux[String, EvaluationFunction => IOChecked[Set[FileEvaluation]]] = at[String]{ _ =>_ => - Set.empty[FileEvaluation].validIOChecked - } -} diff --git a/cwl/src/main/scala/cwl/MyriadOutputTypeToWomType.scala b/cwl/src/main/scala/cwl/MyriadOutputTypeToWomType.scala deleted file mode 100644 index 61107143006..00000000000 --- a/cwl/src/main/scala/cwl/MyriadOutputTypeToWomType.scala +++ /dev/null @@ -1,105 +0,0 @@ -package cwl - -import cats.data.NonEmptyList -import cwl.CwlType.CwlType -import cwl.command.ParentName -import mouse.all._ -import shapeless.Poly1 -import wom.types._ - -object MyriadOutputInnerTypeToString extends Poly1 { - - implicit def cwlType = at[CwlType] { - _.toString - } - - implicit def ors = at[OutputRecordSchema] { - _.toString - } - - implicit def oes = at[OutputEnumSchema] { - _.toString - } - - implicit def oas = at[OutputArraySchema] { - _.toString - } - - implicit def s = at[String] { - identity - } -} - -object MyriadOutputTypeToWomType extends Poly1{ - - import Case._ - - type SchemaDefToWomType = SchemaDefRequirement => WomType - - implicit def cwlType: Aux[MyriadOutputInnerType, SchemaDefToWomType] = at[MyriadOutputInnerType]{ moit => schemaDefRequirement => - moit.fold(MyriadOutputInnerTypeToWomType).apply(schemaDefRequirement) - } - - implicit def acwl: Aux[Array[MyriadOutputInnerType], SchemaDefToWomType] = at[Array[MyriadOutputInnerType]] { types => schemaDefRequirement => - types.partition(_.select[CwlType].contains(CwlType.Null)) match { - // If there's a single non null type, use that - case (Array(), Array(singleNonNullType)) => - singleNonNullType.fold(MyriadOutputInnerTypeToWomType).apply(schemaDefRequirement) - case (Array(), array: Array[MyriadOutputInnerType]) if array.size > 1 => - val types = array.map(_.fold(MyriadOutputInnerTypeToWomType).apply(schemaDefRequirement)) - WomCoproductType(NonEmptyList.fromListUnsafe(types.toList)) - // If there's a null type and a single non null type, it's a WomOptionalType - case (Array(_), Array(singleNonNullType)) => - WomOptionalType(singleNonNullType.fold(MyriadOutputInnerTypeToWomType).apply(schemaDefRequirement)) - case (Array(_), array: Array[MyriadOutputInnerType]) if array.size > 1 => - val types = array.map(_.fold(MyriadOutputInnerTypeToWomType).apply(schemaDefRequirement)) - WomOptionalType(WomCoproductType(NonEmptyList.fromListUnsafe(types.toList))) - case _ => - val readableTypes = types.map(_.fold(MyriadOutputInnerTypeToString)).mkString(", ") - throw new UnsupportedOperationException(s"Cromwell only supports single types or optionals (as indicated by [null, X]). Instead we saw: $readableTypes") - } - } -} - -object MyriadOutputInnerTypeToWomType extends Poly1 { - - import Case._ - import MyriadOutputTypeToWomType.SchemaDefToWomType - - implicit def cwlType: Aux[CwlType, SchemaDefToWomType] = - at[CwlType]{ - cwl.cwlTypeToWomType andThen Function.const - } - - implicit def ors: Aux[OutputRecordSchema, SchemaDefToWomType] = at[OutputRecordSchema] { - ors => schemaDefRequirement => - ors.fields.fold( - WomCompositeType(Map.empty))( - {fields => - val typeMap = fields.map({ field => - val parsedName = FullyQualifiedName(field.name)(ParentName.empty).id - parsedName -> field.`type`.fold(MyriadOutputTypeToWomType).apply(schemaDefRequirement) - }).toMap - WomCompositeType(typeMap) - } - ) - } - - implicit def oes: Aux[OutputEnumSchema, MyriadOutputTypeToWomType.SchemaDefToWomType] = - at[OutputEnumSchema] { - oes => oes.toWomEnumerationType |> Function.const - } - - implicit def oas: Aux[OutputArraySchema, MyriadOutputTypeToWomType.SchemaDefToWomType] = - at[OutputArraySchema] { - oas => schemaDefRequirement => - val arrayType: WomType = oas.items.fold(MyriadOutputTypeToWomType).apply(schemaDefRequirement) - WomArrayType(arrayType) - } - - implicit def s: Aux[String, MyriadOutputTypeToWomType.SchemaDefToWomType] = - at[String] { - string => schemaReq => - schemaReq.lookupType(string).getOrElse(throw new RuntimeException(s"Custom type $string was referred to but not found in schema def ${schemaReq.types.mkString(", ")}.")) - } -} diff --git a/cwl/src/main/scala/cwl/MyriadOutputTypeToWomValue.scala b/cwl/src/main/scala/cwl/MyriadOutputTypeToWomValue.scala deleted file mode 100644 index 332e4ab3888..00000000000 --- a/cwl/src/main/scala/cwl/MyriadOutputTypeToWomValue.scala +++ /dev/null @@ -1,100 +0,0 @@ -package cwl - -import cats.data.Validated.Valid -import cats.syntax.option._ -import cats.syntax.traverse._ -import cats.instances.list._ -import common.validation.IOChecked.IOChecked -import common.validation.IOChecked._ -import cwl.CwlType.CwlType -import cwl.command.ParentName -import mouse.all._ -import shapeless.Poly1 -import wom.types._ -import wom.values.{WomObject, WomValue} - -/** - * Folds a MyriadOutputType into a WomValue - * This is needed because the type might define the structure of the final output value (for OutputRecordSchemas for example) - */ -object MyriadOutputTypeToWomValue extends Poly1 { - - // We pass in a function that can evaluate a CommandOutputBinding and produce a WomValue. This allows us to recurse into the - // MyriadOutputTypes and evaluate values as we do. - type EvaluationFunction = (CommandOutputBinding, WomType) => IOChecked[WomValue] - - //Our overall return type gives us the evaluator function and custom types; returns WomValues - type Output = (EvaluationFunction, SchemaDefRequirement) => IOChecked[WomValue] - - import Case._ - - implicit def cwlType: Aux[MyriadOutputInnerType, Output] = at[MyriadOutputInnerType]{ - moit => (evalFunction, schemaDefRequirement) => moit.fold(MyriadOutputInnerTypeToWomValue).apply(evalFunction, schemaDefRequirement) - } - - // TODO: Not sure what the right thing to do is here, for now go over the list of types and use the first evaluation that yields success - implicit def acwl: Aux[Array[MyriadOutputInnerType], Output] = at[Array[MyriadOutputInnerType]] { types => - (evalFunction,schemaDefRequirement) => - types.toList.map(_.fold(MyriadOutputInnerTypeToWomValue).apply(evalFunction, schemaDefRequirement)).map(_.toErrorOr).collectFirst({ - case Valid(validValue) => validValue - }).toValidNel(s"Cannot find a suitable type to build a WomValue from in ${types.mkString(", ")}").toIOChecked - } -} - -object MyriadOutputInnerTypeToWomValue extends Poly1 { - - import Case._ - import MyriadOutputTypeToWomValue.Output - - def ex(component: String) = throw new RuntimeException(s"output type $component cannot yield a wom value") - - implicit def cwlType: Aux[CwlType, Output] = at[CwlType] { _ => (_,_) => - "No output binding is defined. Are you expecting the output to be inferred from a cwl.output.json file ? If so please make sure the file was effectively created.".invalidIOChecked - } - - implicit def ors: Aux[OutputRecordSchema, Output] = at[OutputRecordSchema] { ors => (evalFunction, schemaDefRequirement) => ors match { - case OutputRecordSchema(_, Some(fields), _) => - // Go over each field and evaluate the binding if there's one, otherwise keep folding over field types - def evaluateValues = fields.toList.traverse[IOChecked, ((String, WomValue), (String, WomType))]({ field => - val womType = field.`type`.fold(MyriadOutputTypeToWomType).apply(schemaDefRequirement) - val womValue: IOChecked[WomValue] = field.outputBinding match { - case Some(binding) => evalFunction(binding, womType) - case None => field.`type`.fold(MyriadOutputTypeToWomValue).apply(evalFunction, schemaDefRequirement) - } - - // TODO: ParentName might need to be passed in here ? - // return the value and the type with a clean parsedName - womValue map { value => - val parsedName = FullyQualifiedName(field.name)(ParentName.empty).id - (parsedName -> value) -> (parsedName -> womType) - } - }) - - evaluateValues map { evaluatedValues => - val (valueMap, typeMap) = evaluatedValues.unzip - // Create a typed WomObject from the values and the typeMap - WomObject.withTypeUnsafe(valueMap.toMap, WomCompositeType(typeMap.toMap)) - } - case ors => ors.toString |> ex - }} - - implicit def oes: Aux[OutputEnumSchema, Output] = at[OutputEnumSchema]{ - //DB: I tried to do a pattern match as the overall function here but the compiler exploded - oes => (f, schemaDefRequirement) => oes match { - case oes@OutputEnumSchema(_, _, _, _, Some(outputBinding)) => f(outputBinding, oes.toWomEnumerationType) - case _ => s"The enumeration type $oes requires an outputbinding to be evaluated.".invalidIOChecked - } - } - - implicit def oas: Aux[OutputArraySchema, Output] = at[OutputArraySchema]{ oas => (evalFunction, schemaDefRequirement) => oas match { - case OutputArraySchema(itemsType, _, _, outputBinding) => - lazy val itemsWomType = itemsType.fold(MyriadOutputTypeToWomType).apply(schemaDefRequirement) - def fromBinding = outputBinding.map(evalFunction(_, WomArrayType(itemsWomType))) - def fromTypes = itemsType.fold(MyriadOutputTypeToWomValue).apply(evalFunction, schemaDefRequirement) - fromBinding.getOrElse(fromTypes) - }} - - implicit def s: Aux[String, Output] = at[String]{ s => (_, _) => - s.toString |> ex - } -} diff --git a/cwl/src/main/scala/cwl/OutputParameter.scala b/cwl/src/main/scala/cwl/OutputParameter.scala deleted file mode 100644 index 274469da7ee..00000000000 --- a/cwl/src/main/scala/cwl/OutputParameter.scala +++ /dev/null @@ -1,62 +0,0 @@ -package cwl - -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.option._ -import common.validation.ErrorOr._ -import shapeless.{:+:, CNil, Poly1} -import wom.values.{WomString, WomValue} - -trait OutputParameter { - def id: String - def label: Option[String] - def secondaryFiles: Option[SecondaryFiles] - def format: Option[OutputParameterFormat] - def streamable: Option[Boolean] - def doc: Option[String :+: Array[String] :+: CNil] - def outputBinding: Option[CommandOutputBinding] - def `type`: Option[MyriadOutputType] - def cacheString: String = toString -} - -object OutputParameter { - object IdAndType { - def unapply(arg: OutputParameter): Option[(String, MyriadOutputType)] = arg.`type`.map((arg.id, _)) - } - - def format(formatOption: Option[StringOrExpression], - parameterContext: ParameterContext, - expressionLib: ExpressionLib): ErrorOr[Option[String]] = { - formatOption.traverse{ - format(_, parameterContext, expressionLib) - } - } - - def format(format: StringOrExpression, parameterContext: ParameterContext, expressionLib: ExpressionLib): ErrorOr[String] = { - format.fold(OutputParameter.FormatPoly).apply(parameterContext, expressionLib) - } - - type FormatFunction = (ParameterContext, ExpressionLib) => ErrorOr[String] - - object FormatPoly extends Poly1 { - implicit def caseStringOrExpression: Case.Aux[StringOrExpression, FormatFunction] = { - at { - _.fold(this) - } - } - - implicit def caseExpression: Case.Aux[Expression, FormatFunction] = { - at { - expression => - (parameterContext, expressionLib) => - val result: ErrorOr[WomValue] = ExpressionEvaluator.eval(expression, parameterContext) - result flatMap { - case womString: WomString => womString.value.valid - case other => s"Not a valid file format: $other".invalidNel - } - } - } - - implicit def caseString: Case.Aux[String, FormatFunction] = at { string => (_,_) => string.valid } - } -} diff --git a/cwl/src/main/scala/cwl/OutputParameterExpression.scala b/cwl/src/main/scala/cwl/OutputParameterExpression.scala deleted file mode 100644 index b044bc36077..00000000000 --- a/cwl/src/main/scala/cwl/OutputParameterExpression.scala +++ /dev/null @@ -1,115 +0,0 @@ -package cwl - -import common.validation.ErrorOr.ErrorOr -import common.validation.IOChecked.{IOChecked, _} -import cwl.CwlType.CwlType -import shapeless.Poly1 -import wom.expression.{EmptyIoFunctionSet, FileEvaluation, IoFunctionSet} -import wom.types._ -import wom.values.WomValue - -import scala.Function.const -import scala.concurrent.{ExecutionContext, Future} - -case class OutputParameterExpression(parameter: OutputParameter, - override val cwlExpressionType: WomType, - override val inputs: Set[String], - override val expressionLib: ExpressionLib, - schemaDefRequirement: SchemaDefRequirement) extends CwlWomExpression { - - override def sourceString = parameter.toString - - override def cacheString: String = parameter.cacheString - - private def evaluateOutputBinding(inputValues: Map[String, WomValue], - ioFunctionSet: IoFunctionSet, - secondaryFilesOption: Option[SecondaryFiles], - formatOption: Option[StringOrExpression] - )(outputBinding: CommandOutputBinding, - cwlExpressionType: WomType): IOChecked[WomValue] = { - CommandOutputBinding.generateOutputWomValue( - inputValues, - ioFunctionSet, - cwlExpressionType, - outputBinding, - secondaryFilesOption, - formatOption, - expressionLib - ) - } - - private def evaluateOutputBindingFiles(inputValues: Map[String, WomValue], - ioFunctionSet: IoFunctionSet, - secondaryFilesOption: Option[SecondaryFiles], - coerceTo: WomType - )(outputBinding: CommandOutputBinding): IOChecked[Set[FileEvaluation]] = { - CommandOutputBinding.getOutputWomFiles( - inputValues, - coerceTo, - outputBinding, - secondaryFilesOption, - ioFunctionSet, - expressionLib - ) - } - - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { - def fromOutputBinding = - parameter.outputBinding.map(evaluateOutputBinding(inputValues, ioFunctionSet, parameter.secondaryFiles, parameter.format)(_, cwlExpressionType)) - - def fromType = - parameter.`type`.map(_.fold(MyriadOutputTypeToWomValue).apply( - evaluateOutputBinding(inputValues, ioFunctionSet, parameter.secondaryFiles, parameter.format), - schemaDefRequirement - )) - - fromOutputBinding.orElse(fromType).getOrElse(s"Cannot evaluate ${parameter.toString}".invalidIOChecked).toErrorOr - } - - /** - * Returns the list of files that _will be_ output after the command is run, unless they are optional and if so _may be_. - * - * In CWL, a list of outputs is specified as glob, say `*.bam`, plus a list of secondary files that may be in the - * form of paths or specified using carets such as `^.bai`. - * - * The coerceTo may be one of four different values: - * - WomMaybePopulatedFileType - * - WomArrayType(WomMaybePopulatedFileType) - * - WomMaybeListedDirectoryType - * - WomArrayType(WomMaybeListedDirectoryType) (Possible according to the way the spec is written, but not likely?) - */ - override def evaluateFiles(inputs: Map[String, WomValue], unused: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { - import cats.syntax.apply._ - - // Ignore the supplied ioFunctionSet and use a custom stubbed IoFunctionSet. This is better than a real I/O function set - // because in the context of file evaluation we don't care about the results of these operations. The NoIoFunctionSet - // that otherwise would be used throws for all of its operations which doesn't fly for the way our CWL evaluation works. - val stubbedIoFunctionSet = new EmptyIoFunctionSet { - override def size(path: String): Future[Long] = Future.successful(0L) - override def isDirectory(path: String): Future[Boolean] = Future.successful(false) - override def ec: ExecutionContext = scala.concurrent.ExecutionContext.global - } - def fromOutputBinding: IOChecked[Set[FileEvaluation]] = parameter - .outputBinding - .map(evaluateOutputBindingFiles(inputs, stubbedIoFunctionSet, parameter.secondaryFiles, coerceTo)) - .getOrElse(Set.empty[FileEvaluation].validIOChecked) - - def fromType: IOChecked[Set[FileEvaluation]] = parameter - .`type` - .map(_.fold(MyriadOutputTypeToWomFiles).apply(evaluateOutputBindingFiles(inputs, stubbedIoFunctionSet, parameter.secondaryFiles, coerceTo))) - .getOrElse(Set.empty[FileEvaluation].validIOChecked) - - val optional: Boolean = parameter.`type`.exists(_.fold(OutputTypeIsOptional)) - - ((fromOutputBinding, fromType) mapN (_ ++ _) map { _ map { _.copy(optional = optional) } }).toErrorOr - } -} - -object OutputTypeIsOptional extends Poly1 { - implicit val one: Case.Aux[MyriadOutputInnerType, Boolean] = at[MyriadOutputInnerType] { const(false) } - - implicit val arr: Case.Aux[Array[MyriadOutputInnerType], Boolean] = at[Array[MyriadOutputInnerType]] { - // Possibly too broad, would return true for just single 'null'. - _.exists(_.select[CwlType].contains(CwlType.Null)) - } -} diff --git a/cwl/src/main/scala/cwl/ParameterContext.scala b/cwl/src/main/scala/cwl/ParameterContext.scala deleted file mode 100644 index 4bcb4d132d5..00000000000 --- a/cwl/src/main/scala/cwl/ParameterContext.scala +++ /dev/null @@ -1,17 +0,0 @@ -package cwl - -import wom.callable.RuntimeEnvironment -import wom.expression.IoFunctionSet -import wom.types.WomNothingType -import wom.values.{WomOptionalValue, WomValue} - -object ParameterContext { - val EmptySelf = WomOptionalValue(WomNothingType, None) -} - -case class ParameterContext( - ioFunctionSet: IoFunctionSet, - expressionLib: ExpressionLib, - inputs: Map[String, WomValue] = Map.empty, - self: WomValue = ParameterContext.EmptySelf, - runtimeOption: Option[RuntimeEnvironment] = None) diff --git a/cwl/src/main/scala/cwl/RequirementsAndHints.scala b/cwl/src/main/scala/cwl/RequirementsAndHints.scala deleted file mode 100644 index 46b5c52e410..00000000000 --- a/cwl/src/main/scala/cwl/RequirementsAndHints.scala +++ /dev/null @@ -1,9 +0,0 @@ -package cwl - -case class RequirementsAndHints(list: List[Requirement]) { - - lazy val hasShellCommandRequirement: Boolean = list.exists(_.select[ShellCommandRequirement].nonEmpty) - - //Should we add up all the types instead? This would mean subworkflows can inherit their parent schemas - lazy val schemaDefRequirement = list.flatMap(_.select[SchemaDefRequirement]).headOption.getOrElse(SchemaDefRequirement()) -} diff --git a/cwl/src/main/scala/cwl/RunOutputsToTypeMap.scala b/cwl/src/main/scala/cwl/RunOutputsToTypeMap.scala deleted file mode 100644 index f327b5b03ea..00000000000 --- a/cwl/src/main/scala/cwl/RunOutputsToTypeMap.scala +++ /dev/null @@ -1,45 +0,0 @@ -package cwl - -import shapeless.Poly1 -import wom.types.WomType - -object RunOutputsToTypeMap extends Poly1 { - - import Case.Aux - - type SchemaDefToTypeMap = SchemaDefRequirement => WomTypeMap - - def handleOutputParameters[A <: OutputParameter](outputs: Array[A], schemaDefRequirement: SchemaDefRequirement): Map[String, WomType] = { - outputs.toList.foldLeft(Map.empty[String, WomType]) { - (acc, out) => - acc ++ - out. - `type`. - map(_.fold(MyriadOutputTypeToWomType).apply(schemaDefRequirement)). - map(out.id -> _). - toList. - toMap - } - } - - implicit def commandLineTool : Aux[CommandLineTool, SchemaDefToTypeMap] = - at[CommandLineTool] { - clt => handleOutputParameters(clt.outputs, _) - } - - implicit def string: Aux[String, SchemaDefToTypeMap] = at[String] { - _ => _ => - Map.empty[String, WomType] //should never happen because we dereference/embed CWL subworkflows into one object from the original string references - } - - implicit def expressionTool: Aux[ExpressionTool, SchemaDefToTypeMap] = at[ExpressionTool] { - et => - handleOutputParameters(et.outputs, _) - } - - implicit def workflow: Aux[Workflow, SchemaDefToTypeMap] = at[Workflow] { - wf => - handleOutputParameters(wf.outputs, _) - } -} - diff --git a/cwl/src/main/scala/cwl/RunToEmbeddedCwl.scala b/cwl/src/main/scala/cwl/RunToEmbeddedCwl.scala deleted file mode 100644 index 2547b6333a4..00000000000 --- a/cwl/src/main/scala/cwl/RunToEmbeddedCwl.scala +++ /dev/null @@ -1,36 +0,0 @@ -package cwl - -import shapeless._ -import WorkflowStep.Run - -object RunToEmbeddedCwl extends Poly1 { - implicit def commandLineTool = - at[CommandLineTool] { - clt => - (_: Map[String, Cwl]) => - Coproduct[Run](clt) - } - - implicit def string = at[String] { - fileName => - (cwlMap: Map[String, Cwl]) => { - val cwl = cwlMap(fileName) - cwl.fold(CwlToRun) - } - } - - - implicit def expressionTool = at[ExpressionTool] { - et => - (_: Map[String, Cwl]) => - Coproduct[Run](et) - } - - implicit def workflow = at[Workflow] { - wf => - (_: Map[String, Cwl]) => - Coproduct[Run](wf) - } -} - - diff --git a/cwl/src/main/scala/cwl/RunToInputTypeMap.scala b/cwl/src/main/scala/cwl/RunToInputTypeMap.scala deleted file mode 100644 index 7111e389050..00000000000 --- a/cwl/src/main/scala/cwl/RunToInputTypeMap.scala +++ /dev/null @@ -1,37 +0,0 @@ -package cwl - -import cwl.command.ParentName -import shapeless.Poly1 - -object RunToInputTypeMap extends Poly1 { - - type MyriadInputTypeMap = Map[String, Option[MyriadInputType]] - - type OutputType = ParentName => MyriadInputTypeMap - - import Case.Aux - - implicit def s: Aux[String, OutputType] = - at[String] { - run => _ => throw new RuntimeException(s"Run field $run was not inlined as expected") - } - - implicit def clt: Aux[CommandLineTool, OutputType] = - at[CommandLineTool]{clt => parentName => clt.inputs.map{ - input => - FullyQualifiedName(input.id)(parentName).id -> input.`type` - }.toMap} - - implicit def et: Aux[ExpressionTool, OutputType] = - at[ExpressionTool]{et => parentName => et.inputs.map{ - input => - FullyQualifiedName(input.id)(parentName).id -> input.`type` - }.toMap} - - implicit def wf: Aux[Workflow, OutputType] = - at[Workflow]{ wf => parentName => wf.inputs.map{ - input => - FullyQualifiedName(input.id)(parentName).id -> input.`type` - }.toMap} - -} diff --git a/cwl/src/main/scala/cwl/ScatterLogic.scala b/cwl/src/main/scala/cwl/ScatterLogic.scala deleted file mode 100644 index 97d87daf6df..00000000000 --- a/cwl/src/main/scala/cwl/ScatterLogic.scala +++ /dev/null @@ -1,263 +0,0 @@ -package cwl - -import cats.data.NonEmptyList -import cats.instances.list._ -import common.Checked -import common.validation.Checked._ -import common.validation.ErrorOr.ErrorOr -import cwl.ScatterMethod.ScatterMethod -import shapeless.Poly1 -import wom.graph.CallNode.CallNodeAndNewNodes -import wom.graph.GraphNodePort.ScatterGathererPort -import wom.graph.ScatterNode._ -import wom.graph._ -import wom.graph.expression.ExpressionNode -import wom.types.{WomArrayType, WomMaybeEmptyArrayType, WomType} -import wom.values.{WomArray, WomValue} - -import scala.annotation.tailrec - -/** - * Contains methods used to implement the scatter related logic for CWL workflows. - * The general idea is as follows: - * - * See WorkflowStep for an example of WOM graph for scatters. - * - * # Scattering over multiple variables - * - * When several variables are being scattered over, a scatter method is mandatory to specify how the elements must be combined. - * Let's use the following 2 arrays as an example: - * - * a1: ["one", "two"], a2: ["three", "four"] - * - * * DotProduct: - * DotProduct only works if all the arrays have the same length. - * - * Shard 0: "one" - "three" - * Shard 1: "two" - "four" - * - * * CrossProduct (nested or flat): - * Both cross product methods generate the same shards, the difference is in how they are collected at the end - * - * Shard 0: "one" - "three" - * Shard 1: "one" - "four" - * Shard 2: "two" - "three" - * Shard 3: "two" - "four" - * - * To support this, each SVN will have a method which given a shard index will return the index at which the value should be looked up in the array. - * - * For dot product, this function is the same for all SVN and will be the identity[Int] function. - * For cross product, this function depends on the number of elements in each array, which is known at runtime. - * When we do have this information we update each SVN with it in the ScatterProcessingFunction and returns the number of shards to be generated. - */ -object ScatterLogic { - case class ScatterFunctions(processing: ScatterProcessingFunction, collection: ScatterCollectionFunctionBuilder) - - object ScatterVariablesPoly extends Poly1 { - implicit def fromString: Case.Aux[String, List[String]] = at[String] { s: String => List(s) } - implicit def fromStringList: Case.Aux[Array[String], List[String]] = at[Array[String]] { l: Array[String] => l.toList } - } - - // Validate that the scatter expression is an ArrayLike and return the member type - private [cwl] def scatterExpressionItemType(expressionNode: ExpressionNode) = { - expressionNode.womType match { - case WomArrayType(itemType) => itemType.validNelCheck // Covers maps because this is a custom unapply (see WomArrayType) - case other => s"Cannot scatter over a non-traversable type ${other.stableName}".invalidNelCheck - } - } - - def scatterGatherPortTypeFunction(scatterMethod: Option[ScatterMethod], scatterVariables: NonEmptyList[_]): WomType => WomArrayType = scatterMethod match { - case Some(ScatterMethod.NestedCrossProduct) => - innerType: WomType => - scatterVariables.tail.foldLeft(WomArrayType(innerType))({ case (t, _) => WomArrayType(t) }) - case _ => innerType: WomType => WomArrayType(innerType) - } - - // Build a map (potentially empty) of scatterered input steps with their corresponding SVN node. - def buildScatterVariableNodes(scatter: ScatterVariables, stepInputMappings: Map[WorkflowStepInput, ExpressionNode], stepId: String): Checked[Map[WorkflowStepInput, ScatterVariableNode]] = { - import cats.syntax.all._ - - def buildScatterVariable(scatterVariableName: String): ErrorOr[(WorkflowStepInput, ScatterVariableNode)] = { - // Assume the variable is a step input (is that always true ??). Find the corresponding expression node - - stepInputMappings.find({ case (stepInput, _) => stepInput.id == scatterVariableName }) match { - case Some((stepInput, expressionNode)) => - scatterExpressionItemType(expressionNode).toValidated map { itemType => - // create a scatter variable node for other scattered nodes to point to - stepInput -> ScatterVariableNode(WomIdentifier(scatterVariableName), expressionNode, itemType) - } - case None => s"Could not find a variable $scatterVariableName in the workflow step input to scatter over. Please make sure $scatterVariableName is an input of step $stepId".invalidNel - } - } - // Take the scatter field defining the (list of) input(s) to be scattered over - scatter - // Fold them so we have a List[String] no matter what (a single scattered input becomes a single element list) - .map(_.fold(ScatterVariablesPoly)) - // If there's no scatter make it an empty list - .getOrElse(List.empty) - // Traverse the list to create ScatterVariableNodes that will be used later on to create the ScatterNode - .traverse(buildScatterVariable) - .toEither - .map(_.toMap) - } - - // Prepare the nodes to be returned if this call is being scattered - def buildScatterNode(callNodeAndNewNodes: CallNodeAndNewNodes, - scatterVariableNodes: NonEmptyList[ScatterVariableNode], - ogins: Set[OuterGraphInputNode], - stepExpressionNodes: Set[ExpressionNode], - scatterMethod: Option[ScatterMethod]): Checked[ScatterNode] = { - - val scatterProcessingFunctionCheck = (scatterVariableNodes.size, scatterMethod) match { - // If we scatter over one variable only, the default processing method can handle it - case (1, _) => ScatterFunctions(ScatterNode.DefaultScatterProcessingFunction, ScatterNode.DefaultScatterCollectionFunctionBuilder).validNelCheck - case (_, Some(method)) => ScatterFunctions(processingFunction(method), collectingFunctionBuilder(method)).validNelCheck - case (_, None) => "When scattering over multiple variables, a scatter method needs to be defined. See http://www.commonwl.org/v1.0/Workflow.html#WorkflowStep".invalidNelCheck - } - - val callNode = callNodeAndNewNodes.node - - // We need to generate PBGONs for every output port of the call, so that they can be linked outside the scatter graph - val portBasedGraphOutputNodes = callNode.outputPorts.map(op => PortBasedGraphOutputNode(op.identifier, op.womType, op)) - - def buildScatterNode(innerGraph: Graph, scatterFunctions: ScatterFunctions) = { - val scatterNodeBuilder = new ScatterNodeBuilder - val outputPorts: Set[ScatterGathererPort] = innerGraph.nodes.collect { case gon: PortBasedGraphOutputNode => - scatterNodeBuilder.makeOutputPort(scatterGatherPortTypeFunction(scatterMethod, scatterVariableNodes)(gon.womType), gon) - } - - scatterNodeBuilder.build(innerGraph, outputPorts, scatterVariableNodes.toList, scatterFunctions.processing, scatterFunctions.collection) - } - - for { - scatterProcessingFunctionAndBuilder <- scatterProcessingFunctionCheck - graph <- Graph.validateAndConstruct( - Set(callNodeAndNewNodes.node) ++ - ogins ++ - stepExpressionNodes ++ - scatterVariableNodes.toList ++ - portBasedGraphOutputNodes - ).toEither - scatterNode = buildScatterNode(graph, scatterProcessingFunctionAndBuilder) - } yield scatterNode.node - } - - /* - * Processing function for cross products. Logic is as follows: - * We want 1) know how many shards should be created (which this function will return) - * 2) update each SVN with their relative index length (see ScatterVariableNode for more detail) - * - * We are going to multiply all the array sizes (which will give us 1) at the end) - * Along the way we will update the relative index length for each SVN. - * To do so, we start at the end of the list and recurse. - */ - private [cwl] val CrossProductScatterProcessingFunction: ScatterProcessingFunction = { nodesAndValues: List[ScatterVariableAndValue] => - @tailrec - def updateIndexLengthRec(currentList: List[ScatterVariableAndValue], combinedArraySize: Int): Int = currentList match { - case Nil => combinedArraySize - case ScatterVariableAndValue(variableNode, arrayValue) :: tail => - variableNode.withRelativeIndexLength(combinedArraySize) - updateIndexLengthRec(tail, combinedArraySize * arrayValue.size) - } - - // Reverse the list so that we start from the last variable and make our way back to the first one - updateIndexLengthRec(nodesAndValues.reverse, 1).validNelCheck - } - - private [cwl] val DotProductScatterProcessingFunction: ScatterProcessingFunction = ScatterNode.DefaultScatterProcessingFunction - - // Recursively nest shard results appropriately based on the size of the scatter variable arrays - private [cwl] val NestedCrossProductScatterCollectionFunctionBuilder: ScatterCollectionFunctionBuilder = { arraySizes: List[Int] => - (sortedShards: List[WomValue], valueType: WomArrayType) => - - @tailrec - def buildCollectorArrayRec(currentList: List[Int], currentWomValues: List[WomValue], currentWomType: WomType): WomArray = { - /* We stop right before the last element because it will necessarily be the number of elements in currentWomValues (provided the list is non empty). - * e.g if currentList is (2, 3, 2), then currentWomValues will have 2 * 3 * 2 = 12 elements (this is a cross product) - * As we recurse, we keep grouping currentWomValues by the numbers in currentList: - * - * group by 2: [[_, _], [_, _], [_, _], [_, _], [_, _], [_, _]] - * group by 3: [ - * [[_, _], [_, _], [_, _]], - * [[_, _], [_, _], [_, _]] - * ] - * This list has necessarily 2 elements already because 12 / 2 / 3 = 2. So we can make the final WomArray out of that - */ - currentList match { - case _ :: Nil | Nil => - WomArray(WomMaybeEmptyArrayType(currentWomType), currentWomValues) - case head :: tail => - val arrayType = WomMaybeEmptyArrayType(currentWomType) - val womArrays = currentWomValues.grouped(head).toList map { WomArray(arrayType, _) } - buildCollectorArrayRec(tail, womArrays, arrayType) - } - } - - def mostInnerType(tpe: WomType): WomType = tpe match { - case WomArrayType(t) => mostInnerType(t) - case other => other - } - - def buildEmptyArrays(arraySizeList: List[Int], womType: WomType, womValues: List[WomValue]): WomArray = arraySizeList match { - case Nil => womType match { - case arrayType: WomArrayType => WomArray(arrayType, womValues) - case _ => - // This would mean arraySizeList was empty to begin with (otherwise we would have wrapped womType in a WomArrayType at least once) - // which should be impossible since it would mean there was no scatter variables, so we shouldn't even be here - throw new RuntimeException("Programmer error ! We should not be collecting scatter nodes if there was no scatter !") - } - case 0 :: tail => - buildEmptyArrays(tail, WomArrayType(womType), List.empty) - case head :: tail => - val arrayType = womType match { - case array: WomArrayType => array - case nonArray => WomArrayType(nonArray) - } - val womArrays = (0 until head).toList map { _ => WomArray(arrayType, womValues) } - buildEmptyArrays(tail, WomArrayType(womType), womArrays) - } - - /* - * There are 2 distinct cases. - * If we have no shards, it means that at least one of the scatter expressions evaluated to an empty array. - * In that case we want to create an array structure of empty arrays reflecting the scatter array sizes. - * e.g: - * - * scatter array sizes | collected result - * [0] [] - * [0, 2] [] - * [2, 0] [[], []] - * [2, 0, 3] [[], []] - * [2, 3, 0] [[[], [], []], [[], [], []]] - * - * Note that as soon as we reach a size 0, we return an empty array. - * - * If we have shards, we successively group the shards list to build arrays of the right size. - * - * In both cases we reverse the list so that we can left recurse while still building the result starting from the most nested arrays - */ - if (sortedShards.isEmpty) { - buildEmptyArrays(arraySizes.reverse, mostInnerType(valueType), List.empty) - } else { - // - buildCollectorArrayRec(arraySizes.reverse, sortedShards, mostInnerType(valueType)) - } - } - - // select a processing function based on the scatter method - private def processingFunction(scatterMethod: ScatterMethod) = scatterMethod match { - case ScatterMethod.DotProduct => DotProductScatterProcessingFunction - // Both cross product methods use the same processing function, the difference is in how we collect results (see collectingFunctionBuilder) - case ScatterMethod.FlatCrossProduct => CrossProductScatterProcessingFunction - case ScatterMethod.NestedCrossProduct => CrossProductScatterProcessingFunction - } - - // select a collecting function builder based on the scatter method - private def collectingFunctionBuilder(scatterMethod: ScatterMethod) = scatterMethod match { - // dot product and flat cross product output a flat array, which is the default behavior - case ScatterMethod.DotProduct => ScatterNode.DefaultScatterCollectionFunctionBuilder - case ScatterMethod.FlatCrossProduct => ScatterNode.DefaultScatterCollectionFunctionBuilder - // nested cross product uses a special collecting function to build nested arrays - case ScatterMethod.NestedCrossProduct => NestedCrossProductScatterCollectionFunctionBuilder - } -} diff --git a/cwl/src/main/scala/cwl/ScatterMethod.scala b/cwl/src/main/scala/cwl/ScatterMethod.scala deleted file mode 100644 index 0a60575e0cb..00000000000 --- a/cwl/src/main/scala/cwl/ScatterMethod.scala +++ /dev/null @@ -1,9 +0,0 @@ -package cwl - -object ScatterMethod extends Enumeration { - type ScatterMethod = Value - - val DotProduct = Value("dotproduct") - val NestedCrossProduct = Value("nested_crossproduct") - val FlatCrossProduct = Value("flat_crossproduct") -} diff --git a/cwl/src/main/scala/cwl/StringOrStringArrayToStringList.scala b/cwl/src/main/scala/cwl/StringOrStringArrayToStringList.scala deleted file mode 100644 index 73afab205b7..00000000000 --- a/cwl/src/main/scala/cwl/StringOrStringArrayToStringList.scala +++ /dev/null @@ -1,12 +0,0 @@ -package cwl - -import shapeless.Poly1 - -/** - * From a logic standpoint it's easier to treat sources as a list of strings. - */ -object StringOrStringArrayToStringList extends Poly1{ - import Case._ - implicit def string: Aux[String, List[String]] = at[String]{List(_)} - implicit def array: Aux[Array[String], List[String]] = at[Array[String]]{_.toList} -} diff --git a/cwl/src/main/scala/cwl/Tool.scala b/cwl/src/main/scala/cwl/Tool.scala deleted file mode 100644 index d31b7368aee..00000000000 --- a/cwl/src/main/scala/cwl/Tool.scala +++ /dev/null @@ -1,189 +0,0 @@ -package cwl - -import java.nio.file.Paths - -import cats.instances.list._ -import common.Checked -import common.validation.ErrorOr.ErrorOr -import common.validation.Validation._ -import cwl.CwlVersion.CwlVersion -import cwl.Tool.inlineJavascriptRequirements -import cwl.command.ParentName -import cwl.requirement.RequirementToAttributeMap -import shapeless.Inl -import wom.callable.Callable.{OverridableInputDefinitionWithDefault, OptionalInputDefinition, OutputDefinition, RequiredInputDefinition} -import wom.callable.MetaValueElement.{MetaValueElementBoolean, MetaValueElementObject} -import wom.callable.{Callable, MetaValueElement, TaskDefinition} -import wom.executable.Executable -import wom.expression.{IoFunctionSet, ValueAsAnExpression, WomExpression} -import wom.types.WomOptionalType -import wom.values.{WomInteger, WomLong, WomString} -import wom.{RuntimeAttributes, RuntimeAttributesKeys} - -import scala.util.Try - -object Tool { - def inlineJavascriptRequirements(allRequirementsAndHints: Seq[Requirement]): Vector[String] = { - val inlineJavscriptRequirements: Seq[InlineJavascriptRequirement] = allRequirementsAndHints.toList.collect { - case Inl(ijr:InlineJavascriptRequirement) => ijr - } - - inlineJavscriptRequirements.flatMap(_.expressionLib.toList.flatten).toVector - } -} - -/** - * Abstraction over a Tool (CommandLineTool or ExpressionTool) - * Contains common logic to both, mostly to build the wom task definition. - */ -trait Tool { - def inputs: Array[_ <: InputParameter] - def outputs: Array[_ <: OutputParameter] - def `class`: String - def id: String - def requirements: Option[Array[Requirement]] - def hints: Option[Array[Hint]] - def label: Option[String] - def doc: Option[String] - def cwlVersion: Option[CwlVersion] - - def asCwl: Cwl - - /** Builds an `Executable` directly from a `Tool` CWL with no parent workflow. */ - def womExecutable(validator: RequirementsValidator, inputFile: Option[String] = None, ioFunctions: IoFunctionSet, strictValidation: Boolean): Checked[Executable] = { - val taskDefinition = buildTaskDefinition(validator, Vector.empty) - CwlExecutableValidation.buildWomExecutable(taskDefinition, inputFile, ioFunctions, strictValidation) - } - - protected def buildTaskDefinition(taskName: String, - inputDefinitions: List[_ <: Callable.InputDefinition], - outputDefinitions: List[Callable.OutputDefinition], - runtimeAttributes: RuntimeAttributes, - requirementsAndHints: List[cwl.Requirement], - expressionLib: ExpressionLib): ErrorOr[TaskDefinition] - - private [cwl] implicit val explicitWorkflowName = ParentName(id) - protected val inputNames: Set[String] = this.inputs.map(i => FullyQualifiedName(i.id).id).toSet - - // Circe can't create bidirectional links between workflow steps and runs (including `CommandLineTool`s) so this - // ugly var is here to link back to a possible parent workflow step. This is needed to navigate upward for finding - // requirements in the containment hierarchy. There isn't always a containing workflow step so this is an `Option`. - private[cwl] var parentWorkflowStep: Option[WorkflowStep] = None - - - private def validateRequirementsAndHints(validator: RequirementsValidator): ErrorOr[List[Requirement]] = { - import cats.syntax.traverse._ - - val allRequirements = requirements.toList.flatten ++ parentWorkflowStep.toList.flatMap(_.allRequirements.list) - // All requirements must validate or this fails. - val errorOrValidatedRequirements: ErrorOr[List[Requirement]] = allRequirements traverse validator - - errorOrValidatedRequirements map { validRequirements => - // Only Requirement hints, everything else is thrown out. - // TODO CWL don't throw them out but pass them back to the caller to do with as the caller pleases. - val hintRequirements = hints.toList.flatten.flatMap { _.select[Requirement] } - val parentHintRequirements = parentWorkflowStep.toList.flatMap(_.allHints) - - // Throw out invalid Requirement hints. - // TODO CWL pass invalid hints back to the caller to do with as the caller pleases. - val validHints = (hintRequirements ++ parentHintRequirements).collect { case req if validator(req).isValid => req } - validRequirements ++ validHints - } - } - - private def processRequirement(requirement: Requirement, expressionLib: ExpressionLib): Map[String, WomExpression] = { - requirement.fold(RequirementToAttributeMap).apply(inputNames, expressionLib) - } - - protected def toolAttributes: Map[String, WomExpression] = Map.empty - - def buildTaskDefinition(validator: RequirementsValidator, parentExpressionLib: ExpressionLib): Checked[TaskDefinition] = { - def build(requirementsAndHints: Seq[cwl.Requirement]) = { - val id = this.id - - val expressionLib: ExpressionLib = parentExpressionLib ++ inlineJavascriptRequirements(requirementsAndHints) - - // This is basically doing a `foldMap` but can't actually be a `foldMap` because: - // - There is no monoid instance for `WomExpression`s. - // - We want to fold from the right so the hints and requirements with the lowest precedence are processed first - // and later overridden if there are duplicate hints or requirements of the same type with higher precedence. - val attributesMap: Map[String, WomExpression] = requirementsAndHints.foldRight(Map.empty[String, WomExpression])({ - case (requirement, acc) => acc ++ processRequirement(requirement, expressionLib) - }) - - val runtimeAttributes: RuntimeAttributes = RuntimeAttributes(attributesMap ++ toolAttributes) - - val schemaDefRequirement: SchemaDefRequirement = requirementsAndHints.flatMap{ - _.select[SchemaDefRequirement].toList - }.headOption.getOrElse(SchemaDefRequirement()) - - lazy val localizationOptionalMetaObject: MetaValueElement = MetaValueElementObject( - Map( - "localization_optional" -> MetaValueElementBoolean(true) - ) - ) - - // If input dir min == 0 that's a signal not to localize the input files - // https://github.com/dnanexus/dx-cwl/blob/fca163d825beb62f8a3004f2e0a6742805e6218c/dx-cwl#L426 - val localizationOptional = runtimeAttributes.attributes.get(RuntimeAttributesKeys.DnaNexusInputDirMinKey) match { - case Some(ValueAsAnExpression(WomInteger(0))) => Option(localizationOptionalMetaObject) - case Some(ValueAsAnExpression(WomLong(0L))) => Option(localizationOptionalMetaObject) - case Some(ValueAsAnExpression(WomString("0"))) => Option(localizationOptionalMetaObject) - case _ => None - } - - val inputDefinitions: List[_ <: Callable.InputDefinition] = - this.inputs.map { - case input @ InputParameter.IdDefaultAndType(inputId, default, tpe) => - val inputType = tpe.fold(MyriadInputTypeToWomType).apply(schemaDefRequirement) - val inputName = FullyQualifiedName(inputId).id - val defaultWomValue = default.fold(InputParameter.DefaultToWomValuePoly).apply(inputType).toTry.get - OverridableInputDefinitionWithDefault( - inputName, - inputType, - ValueAsAnExpression(defaultWomValue), - InputParameter.inputValueMapper(input, tpe, expressionLib, asCwl.schemaOption), - localizationOptional - ) - case input @ InputParameter.IdAndType(inputId, tpe) => - val inputType = tpe.fold(MyriadInputTypeToWomType).apply(schemaDefRequirement) - val inputName = FullyQualifiedName(inputId).id - inputType match { - case optional: WomOptionalType => - OptionalInputDefinition( - inputName, - optional, - InputParameter.inputValueMapper(input, tpe, expressionLib, asCwl.schemaOption), - localizationOptional - ) - case _ => - RequiredInputDefinition( - inputName, - inputType, - InputParameter.inputValueMapper(input, tpe, expressionLib, asCwl.schemaOption), - localizationOptional - ) - } - case other => throw new UnsupportedOperationException(s"command input parameters such as $other are not yet supported") - }.toList - - val outputDefinitions: List[Callable.OutputDefinition] = this.outputs.map { - case p @ OutputParameter.IdAndType(cop_id, tpe) => - val womType = tpe.fold(MyriadOutputTypeToWomType).apply(schemaDefRequirement) - OutputDefinition(FullyQualifiedName(cop_id).id, womType, OutputParameterExpression(p, womType, inputNames, expressionLib, schemaDefRequirement)) - case other => throw new UnsupportedOperationException(s"Command output parameters such as $other are not yet supported") - }.toList - - // The try will succeed if this is a task within a step. If it's a standalone file, the ID will be the file, - // so the filename is the fallback. - def taskName = Try(FullyQualifiedName(id).id).getOrElse(Paths.get(id).getFileName.toString) - - buildTaskDefinition(taskName, inputDefinitions, outputDefinitions, runtimeAttributes, requirementsAndHints.toList, expressionLib).toEither - } - - validateRequirementsAndHints(validator).toEither match { - case Right(requirementsAndHints: Seq[cwl.Requirement]) => build(requirementsAndHints) - case Left(errors) => Left(errors) - } - } -} diff --git a/cwl/src/main/scala/cwl/TypeAliases.scala b/cwl/src/main/scala/cwl/TypeAliases.scala deleted file mode 100644 index abbfc9e81e0..00000000000 --- a/cwl/src/main/scala/cwl/TypeAliases.scala +++ /dev/null @@ -1,115 +0,0 @@ -package cwl - -import cwl.CwlType.CwlType -import cwl.ExpressionEvaluator.{ECMAScriptExpression, ECMAScriptFunction} -import io.circe.Json -import shapeless.{:+:, CNil} -import wom.types.WomType - -trait TypeAliases { - - type Expression = ECMAScriptFunction :+: ECMAScriptExpression :+: CNil - - // http://www.commonwl.org/v1.0/Workflow.html#InputParameter - // http://www.commonwl.org/v1.0/CommandLineTool.html#CommandInputParameter - type InputParameterFormat = Expression :+: String :+: Array[String] :+: CNil - - // http://www.commonwl.org/v1.0/Workflow.html#ExpressionToolOutputParameter - // http://www.commonwl.org/v1.0/CommandLineTool.html#CommandOutputParameter - type OutputParameterFormat = StringOrExpression - - type Doc = String :+: Array[String] :+: CNil - - type StringOrExpression = Expression :+: String :+: CNil - - type CwlAny = - FileOrDirectory :+: - Array[FileOrDirectory] :+: - Json :+: - CNil - - type WorkflowStepInputId = String - - type Requirement = - InlineJavascriptRequirement :+: - SchemaDefRequirement :+: - DockerRequirement :+: - SoftwareRequirement :+: - InitialWorkDirRequirement :+: - EnvVarRequirement :+: - ShellCommandRequirement :+: - ResourceRequirement :+: - SubworkflowFeatureRequirement :+: - ScatterFeatureRequirement :+: - MultipleInputFeatureRequirement :+: - StepInputExpressionRequirement :+: - DnaNexusInputResourceRequirement :+: - CNil - - type Hint = - Requirement :+: - CwlAny :+: - CNil - - type MyriadInputType = - MyriadInputInnerType :+: - Array[MyriadInputInnerType] :+: - CNil - - type MyriadInputInnerType = - CwlType :+: - InputRecordSchema :+: - InputEnumSchema :+: - InputArraySchema :+: - String :+: - CNil - - type MyriadOutputType = - MyriadOutputInnerType :+: - Array[MyriadOutputInnerType] :+: - CNil - - type MyriadOutputInnerType = - CwlType :+: - OutputRecordSchema :+: - OutputEnumSchema :+: - OutputArraySchema :+: - String :+: - CNil - - type ResourceRequirementType = Long :+: Expression :+: String :+: CNil - - type SingleOrArrayOfStrings = String :+: Array[String] :+: CNil - - type ScatterVariables = Option[SingleOrArrayOfStrings] - - type FileOrDirectory = File :+: Directory :+: CNil - - type Glob = StringOrExpression :+: Array[String] :+: CNil - - type SecondaryFiles = StringOrExpression :+: Array[StringOrExpression] :+: CNil -} - -object MyriadInputType { - - object InputArraySchema { - def unapply(m: MyriadInputType): Option[InputArraySchema] = m.select[MyriadInputInnerType].flatMap(_.select[InputArraySchema]) - } - - object InputArray { - def unapply(m: MyriadInputType): Option[Array[MyriadInputInnerType]] = m.select[Array[MyriadInputInnerType]] - } - - object CwlType { - def unapply(m: MyriadInputType): Option[CwlType] = { - m.select[MyriadInputInnerType].flatMap(_.select[CwlType]) - } - } - - object WomType { - def unapply(m: MyriadInputType): Option[WomType] = m match { - case CwlType(c) => Option(cwl.cwlTypeToWomType(c)) - case _ => None - } - } -} diff --git a/cwl/src/main/scala/cwl/Workflow.scala b/cwl/src/main/scala/cwl/Workflow.scala deleted file mode 100644 index 22b30019ff1..00000000000 --- a/cwl/src/main/scala/cwl/Workflow.scala +++ /dev/null @@ -1,246 +0,0 @@ -package cwl - -import java.nio.file.Paths - -import cats.data.NonEmptyList -import cats.syntax.either._ -import cats.syntax.traverse._ -import cats.instances.list._ -import common.Checked -import common.validation.ErrorOr._ -import common.validation.Validation._ -import cwl.CwlVersion._ -import cwl.LinkMergeMethod.LinkMergeMethod -import cwl.Workflow.{WorkflowInputParameter, WorkflowOutputParameter} -import cwl.command.ParentName -import shapeless._ -import shapeless.syntax.singleton._ -import wom.SourceFileLocation -import wom.callable.{MetaValueElement, WorkflowDefinition} -import wom.executable.Executable -import wom.expression.{IoFunctionSet, ValueAsAnExpression} -import wom.graph.GraphNodePort.{GraphNodeOutputPort, OutputPort} -import wom.graph._ -import wom.types.{WomOptionalType, WomType} - -case class Workflow private( - cwlVersion: Option[CwlVersion], - `class`: Witness.`"Workflow"`.T, - id: String, - inputs: Array[WorkflowInputParameter], - outputs: Array[WorkflowOutputParameter], - steps: Array[WorkflowStep], - requirements: Option[Array[Requirement]], - hints: Option[Array[Hint]], - `$namespaces`: Option[Map[String, String]], - `$schemas`: Option[Array[String]] - ) { - - steps.foreach { _.parentWorkflow = this } - - /** Builds an `Executable` from a `Workflow` CWL with no parent `Workflow` */ - def womExecutable(validator: RequirementsValidator, inputFile: Option[String] = None, ioFunctions: IoFunctionSet, strictValidation: Boolean): Checked[Executable] = { - CwlExecutableValidation.buildWomExecutableCallable(womDefinition(validator, Vector.empty), inputFile, ioFunctions, strictValidation) - } - - /* Circe can't create bidirectional links between workflow steps and runs (including `Workflow`s) so this - * ugly var is here to link back to a possible parent workflow step. This is needed to navigate upward for finding - * requirements in the containment hierarchy. There isn't always a containing workflow step so this is an `Option`. - */ - private[cwl] var parentWorkflowStep: Option[WorkflowStep] = None - - val allRequirements: RequirementsAndHints = RequirementsAndHints(requirements.toList.flatten ++ parentWorkflowStep.toList.flatMap { _.allRequirements.list }) - - private [cwl] implicit val explicitWorkflowName = ParentName(id) - - lazy val womFqn: Option[wom.graph.FullyQualifiedName] = { - explicitWorkflowName.value map { workflowName => - /* Sometimes the workflow name is of the form "parentWorkflowStepName/UUID" - * In that case, we don't want the fqn to look like "parentWorkflowStepName.parentWorkflowStepName/UUID" - * To avoid that, strip the parentWorkflowStepName from the workflowName - */ - val cleanWorkflowName = parentWorkflowStep - .map(_.womFqn.value + "/") - .map(workflowName.stripPrefix) - .getOrElse(workflowName) - - parentWorkflowStep.map(_.womFqn.combine(cleanWorkflowName)) - .getOrElse(wom.graph.FullyQualifiedName(cleanWorkflowName)) - } - } - - lazy val allHints: List[Requirement] = { - // Just ignore any hint that isn't a Requirement. - val requirementHints = hints.toList.flatten.flatMap { _.select[Requirement] } - requirementHints ++ parentWorkflowStep.toList.flatMap { _.allHints } - } - - val fileNames: List[String] = steps.toList.flatMap(_.run.select[String].toList) - - def outputsTypeMap: WomTypeMap = steps.foldLeft(Map.empty[String, WomType]) { - /* Not implemented as a `foldMap` because there is no semigroup instance for `WomType`s. `foldMap` doesn't know that - * we don't need a semigroup instance since the map keys should be unique and therefore map values would never need - * to be combined under the same key. - */ - (acc, s) => acc ++ s.typedOutputs - } - - def womGraph(workflowName: String, validator: RequirementsValidator, expressionLib: ExpressionLib): Checked[Graph] = { - val workflowNameIdentifier = explicitWorkflowName.value.map(WomIdentifier.apply).getOrElse(WomIdentifier(workflowName)) - - def womTypeForInputParameter(input: InputParameter): Option[WomType] = { - input.`type`.map(_.fold(MyriadInputTypeToWomType).apply(allRequirements.schemaDefRequirement)) - } - - val typeMap: WomTypeMap = - outputsTypeMap ++ - /* Note this is only looking at the workflow inputs and not recursing into steps, because our current thinking - * is that in CWL graph inputs can only be defined at the workflow level. It's possible that's not actually - * correct, but that's the assumption being made here. - */ - inputs.toList.flatMap { i => - womTypeForInputParameter(i).map(i.id -> _).toList - }.toMap - - val externalGraphInputNodes: Set[ExternalGraphInputNode] = inputs.map { wip => - val womType: WomType = womTypeForInputParameter(wip).get - val parsedInputId = FileAndId(wip.id).id - val womId = WomIdentifier(parsedInputId, wip.id) - val valueMapper = - InputParameter.inputValueMapper(wip, wip.`type`.get, expressionLib, asCwl.schemaOption) - - def optionalWithDefault(memberType: WomType): OptionalGraphInputNodeWithDefault = { - val defaultValue = wip.default.get.fold(InputParameter.DefaultToWomValuePoly).apply(womType).toTry.get - OptionalGraphInputNodeWithDefault(womId, memberType, ValueAsAnExpression(defaultValue), parsedInputId, valueMapper) - } - - womType match { - case WomOptionalType(memberType) if wip.default.isDefined => optionalWithDefault(memberType) - case _ if wip.default.isDefined => optionalWithDefault(womType) - case optional @ WomOptionalType(_) => OptionalGraphInputNode(womId, optional, parsedInputId, valueMapper) - case _ => RequiredGraphInputNode(womId, womType, parsedInputId, valueMapper) - } - }.toSet - - val workflowInputs: Map[String, GraphNodeOutputPort] = - externalGraphInputNodes.map { egin => - egin.localName -> egin.singleOutputPort - }.toMap - - val graphFromSteps: Checked[Set[GraphNode]] = - steps. - toList. - foldLeft((Set.empty[GraphNode] ++ externalGraphInputNodes).asRight[NonEmptyList[String]])( - (nodes, step) => nodes.flatMap(step.callWithInputs(typeMap, this, _, workflowInputs, validator, expressionLib))) - - val graphFromOutputs: Checked[Set[GraphNode]] = - outputs.toList.traverse[ErrorOr, GraphNode] { - case WorkflowOutputParameter(id, _, _, _, _, _, _, Some(Inl(outputSource: String)), _, Some(tpe)) => - val womType: WomType = tpe.fold(MyriadOutputTypeToWomType).apply(allRequirements.schemaDefRequirement) - - val parsedWorkflowOutput = FileAndId(id) - val parsedOutputSource = cwl.FullyQualifiedName(outputSource) - - // Try to find an output port for this cwl output in the set of available nodes - def lookupOutputSource(fqn: cwl.FullyQualifiedName): Checked[OutputPort] = { - def isRightOutputPort(op: GraphNodePort.OutputPort) = cwl.FullyQualifiedName.maybeApply(op.name) match { - case Some(f) => f.id == fqn.id - case None => op.internalName == fqn.id - } - - def sourceNode(graph: Set[GraphNode]): Checked[GraphNode] = { - val findSource: PartialFunction[GraphNode, GraphNode] = fqn match { - // A step output becoming a workflow output. - case fsi: FileStepAndId => { - case callNode: CallNode if callNode.localName == fsi.stepId => callNode - case scatterNode: ScatterNode if scatterNode.innerGraph.calls.exists(_.localName == fsi.stepId) => scatterNode - } - // A workflow input recycled back to be an output. - case fi: FileAndId => { - case gin: ExternalGraphInputNode if gin.nameInInputSet == fi.id => gin - } - } - graph collectFirst findSource toRight NonEmptyList.one(s"Call Node by name $fqn was not found in set $graph") - } - - for { - set <- graphFromSteps - node <- sourceNode(set) - output <- node.outputPorts.find(isRightOutputPort).toChecked(s"looking for ${fqn.id} in call $node output ports ${node.outputPorts}") - } yield output - } - - lookupOutputSource(parsedOutputSource).map({ port => - val localName = LocalName(parsedWorkflowOutput.id) - val fullyQualifiedName = workflowNameIdentifier.fullyQualifiedName.combine(parsedWorkflowOutput.id) - val outputIdentifier = WomIdentifier(localName, fullyQualifiedName) - PortBasedGraphOutputNode(outputIdentifier, womType, port) - }).toValidated - case wop => throw new UnsupportedOperationException(s"Workflow output parameters such as $wop are not supported.") - }.map(_.toSet).toEither - - for { - outputs <- graphFromOutputs - steps <- graphFromSteps - ret <- Graph.validateAndConstruct(steps ++ externalGraphInputNodes ++ outputs).toEither - } yield ret - } - - def womDefinition(validator: RequirementsValidator, expressionLib: ExpressionLib): Checked[WorkflowDefinition] = { - val name: String = Paths.get(id).getFileName.toString - val meta: Map[String, MetaValueElement.MetaValueElementString] = Map.empty - val paramMeta: Map[String, MetaValueElement.MetaValueElementString] = Map.empty - val lexInfo : Option[SourceFileLocation] = None - - womGraph(name, validator, expressionLib).map(graph => - WorkflowDefinition( - name, - graph, - meta, - paramMeta, - lexInfo - ) - ) - } - - def asCwl = Coproduct[Cwl](this) -} -object Workflow { - - case class WorkflowInputParameter(id: String, - label: Option[String] = None, - secondaryFiles: Option[SecondaryFiles] = None, - format: Option[InputParameterFormat] = None, - streamable: Option[Boolean] = None, - doc: Option[Doc] = None, - inputBinding: Option[InputCommandLineBinding] = None, - default: Option[CwlAny] = None, - `type`: Option[MyriadInputType] = None) extends InputParameter - - case class WorkflowOutputParameter( - id: String, - label: Option[String] = None, - secondaryFiles: Option[SecondaryFiles] = None, - format: Option[OutputParameterFormat] = None, - streamable: Option[Boolean] = None, - doc: Option[Doc] = None, - outputBinding: Option[CommandOutputBinding] = None, - outputSource: Option[WorkflowOutputParameter#OutputSource] = None, - linkMerge: Option[LinkMergeMethod] = None, - `type`: Option[MyriadOutputType] = None) extends OutputParameter { - - type OutputSource = String :+: Array[String] :+: CNil - } - - def apply(cwlVersion: Option[CwlVersion] = Option(CwlVersion.Version1), - id: String, - inputs: Array[WorkflowInputParameter] = Array.empty, - outputs: Array[WorkflowOutputParameter] = Array.empty, - steps: Array[WorkflowStep] = Array.empty, - requirements: Option[Array[Requirement]] = None, - hints: Option[Array[Hint]] = None, - namespaces: Option[Map[String, String]] = None, - schemas: Option[Array[String]] = None - ): Workflow = - Workflow(cwlVersion, "Workflow".narrow, id, inputs, outputs, steps, requirements, hints, namespaces, schemas) -} diff --git a/cwl/src/main/scala/cwl/WorkflowStep.scala b/cwl/src/main/scala/cwl/WorkflowStep.scala deleted file mode 100644 index cf39f261bae..00000000000 --- a/cwl/src/main/scala/cwl/WorkflowStep.scala +++ /dev/null @@ -1,537 +0,0 @@ -package cwl - -import cats.Monoid -import cats.data.NonEmptyList -import cats.data.Validated._ -import cats.syntax.either._ -import cats.syntax.foldable._ -import cats.syntax.monoid._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import common.Checked -import common.validation.Checked._ -import common.validation.ErrorOr.ErrorOr -import cwl.ScatterLogic.ScatterVariablesPoly -import cwl.ScatterMethod._ -import cwl.WorkflowStep.{WorkflowStepInputFold, _} -import cwl.WorkflowStepInput._ -import cwl.command.ParentName -import shapeless.{:+:, CNil, _} -import wom.callable.Callable -import wom.callable.Callable._ -import wom.graph.CallNode._ -import wom.graph.GraphNodePort.{GraphNodeOutputPort, OutputPort} -import wom.graph._ -import wom.graph.expression.ExpressionNode -import wom.types.WomType -import wom.values.WomValue -import wom.graph.{FullyQualifiedName => WomFullyQualifiedName} - -/** - * An individual job to run. - * - * @see CWL Spec | Workflow Step - * @param run Purposefully not defaulted as it's required in the specification and it is unreasonable to not have something to run. - */ -case class WorkflowStep( - id: String, - in: Array[WorkflowStepInput] = Array.empty, - out: WorkflowStepOutputType, - run: Run, - requirements: Option[Array[Requirement]] = None, - hints: Option[Array[Hint]] = None, - label: Option[String] = None, - doc: Option[String] = None, - scatter: ScatterVariables = None, - scatterMethod: Option[ScatterMethod] = None) { - - run.select[Workflow].foreach(_.parentWorkflowStep = Option(this)) - run.select[CommandLineTool].foreach(_.parentWorkflowStep = Option(this)) - run.select[ExpressionTool].foreach(_.parentWorkflowStep = Option(this)) - - // We're scattering if scatter is defined, and if it's a list of variables the list needs to be non empty - private val isScattered: Boolean = scatter exists { _.select[Array[String]].forall(_.nonEmpty) } - - // Circe can't create bidirectional links between workflows and workflow steps so this ugly var is here to link - // back to the parent workflow. This is needed to navigate upward for finding requirements in the containment - // hierarchy. There is always a workflow containing a workflow step so this is not an `Option`. - private[cwl] var parentWorkflow: Workflow = _ - - lazy val allRequirements = RequirementsAndHints(requirements.toList.flatten ++ parentWorkflow.allRequirements.list) - - lazy val womFqn: WomFullyQualifiedName = { - implicit val parentName = parentWorkflow.explicitWorkflowName - val localFqn = cwl.FullyQualifiedName.maybeApply(id).map(_.id).getOrElse(id) - parentWorkflow.womFqn.map(_.combine(localFqn)).getOrElse(wom.graph.FullyQualifiedName(localFqn)) - } - - lazy val allHints: List[Requirement] = { - // Just ignore any hint that isn't a Requirement. - val requirementHints = hints.toList.flatten.flatMap { _.select[Requirement] } - requirementHints ++ parentWorkflow.allHints - } - - // If the step is being scattered over, apply the necessary transformation to get the final output array type. - lazy val scatterTypeFunction: WomType => WomType = scatter.map(_.fold(ScatterVariablesPoly)) match { - case Some(Nil) => identity[WomType] - case Some(nonEmpty) => ScatterLogic.scatterGatherPortTypeFunction(scatterMethod, NonEmptyList.fromListUnsafe(nonEmpty)) - case _ => identity[WomType] - } - - def typedOutputs: WomTypeMap = { - implicit val parentName = ParentName(id) - // Find the type of the outputs of the run section - val runOutputTypes = run.fold(RunOutputsToTypeMap).apply(allRequirements.schemaDefRequirement) - .map({ - case (runOutputId, womType) => cwl.FullyQualifiedName(runOutputId).id -> womType - }) - // Use them to find get the final type of the workflow outputs, and only the workflow outputs - out.map({ stepOutput => - val stepOutputValue = stepOutput.select[WorkflowStepOutput].map(_.id).getOrElse(stepOutput.select[String].get) - val stepOutputId = cwl.FullyQualifiedName(stepOutputValue) - stepOutputValue -> scatterTypeFunction(runOutputTypes(stepOutputId.id)) - }).toMap - } - - def fileName: Option[String] = run.select[String] - - /** - * Generates all GraphNodes necessary to represent call nodes and input nodes - * Recursive because dependencies are discovered as we iterate through inputs and corresponding - * upstream nodes need to be generated on the fly. - * - * Example: - * - * CWL: - * inputs: - * 
-id: workflow_input_A
 - * type: string[] - * -id: workflow_input_B
 - * type: string[] - * 
-id: workflow_input_C
 - * type: string

 - * steps:
 - * -id: echo - * run: echo.cwl - * scatter: input0 - * in: - * -id:input0 - * source: - * - "#workflow_input_A" - * - "#workflow_input_B"
 - * valueFrom:"bonjour" - * - * -id: input1 - * 
 source: "#workflow_input_C" - * valueFrom:"$(inputs.input0)" - * - * WOM Diagram: - * - * +------------+ +------------+ +------------+ - * | WF Input A | | WF Input B | | WF Input C | - * +--------+---+ +-+------------+ +------+-----+ - * | | | - * | | | - * +-----v-----v+ +------v-----+ - * | StepInput0 | | StepInput1 | - * | MergeLogic | | MergeLogic | - * +-----+------+ +----+-------+ - * | | - * +----------------------------------------------+ - * | | | | - * | +------v----------+ +---v--+ | - * | | ScatterVariable +------------+ | OGIN | | - * | +-----+-----------+ | +--+-+-+ | - * | | | | | | - * | +-----v----+ | | | | - * | |StepInput0<------------------------+ | | - * | |Expression| | | | - * | +----+-----+ +--v------v+ | - * | | |StepInput1| | - * | | |Expression| | - * | | +------+---+ | - * | | +-----------+ | | - * | +----> Call Node <------------+ | - * | +-----------+ | - * | Scatter Node | - * +----------------------------------------------+ - * - * MergeNode: If the step input has one or more sources, a merge node will be created and responsible for merging - * those input sources together. It will NOT evaluate the valueFrom field of the input. - * - * ScatterVariableNode: If the step input is being scattered over, a scatter variable node will be created and will - * act as a proxy inside the scatter graph for the shards of the scatter. It depends on an upstream merge node outputing an array - * and will provide at runtime shard values for other nodes of the scatter graph. - * - * OGIN: If the step has at least one input being scattered over, there will be a scatter node created. - * For inputs that are NOT being scattered over but still have one or more input sources (and hence a merge node), an OGIN - * will be created to act as a proxy to the merge node outside the scatter graph. - * - * ExpressionNode: If an input has a valueFrom field, an expression node will be created to evaluate the expression. - * An important fact to note is that the expression needs access to all other input values - * AFTER their source, default value and shard number has been determined but - * BEFORE their (potential) valueFrom is evaluated (see http://www.commonwl.org/v1.0/Workflow.html#WorkflowStepInput) - * This is why on the above diagram, StepInput0Expression depends on the OGIN, and StepInput1Expression depends on the scatter variable. - */ - def callWithInputs(typeMap: WomTypeMap, - workflow: Workflow, - knownNodes: Set[GraphNode], - workflowInputs: Map[String, GraphNodeOutputPort], - validator: RequirementsValidator, - expressionLib: ExpressionLib): Checked[Set[GraphNode]] = { - - implicit val parentName = workflow.explicitWorkflowName - - val scatterLookupSet = - scatter.toList. - flatMap(_.fold(StringOrStringArrayToStringList)). - map(id => cwl.FullyQualifiedName(id).id) - - def isStepScattered(workflowStepInputId: String) = scatterLookupSet.contains(workflowStepInputId) - - val unqualifiedStepId: WomIdentifier = { - cwl.FullyQualifiedName.maybeApply(id).map({ fqn => - WomIdentifier(LocalName(fqn.id), womFqn) - }).getOrElse(WomIdentifier(id)) - } - - def typedRunInputs: Map[String, Option[MyriadInputType]] = run.fold(RunToInputTypeMap).apply(parentName) - - def allIdentifiersRecursively(nodes: Set[GraphNode]): Set[WomIdentifier] = nodes.flatMap({ - case w: WorkflowCallNode=> Set(w.identifier) - case c: CommandCallNode => Set(c.identifier) - case e: ExpressionCallNode => Set(e.identifier) - // When a node a call node is being scattered over, it is wrapped inside a scatter node. We still don't want to - // duplicate it though so look inside scatter nodes to see if it's there. - case scatter: ScatterNode => allIdentifiersRecursively(scatter.innerGraph.nodes) - case _ => Set.empty[WomIdentifier] - }) - - // To avoid duplicating nodes, return immediately if we've already covered this node - val haveWeSeenThisStep: Boolean = allIdentifiersRecursively(knownNodes).contains(unqualifiedStepId) - - if (haveWeSeenThisStep) Right(knownNodes) - else { - val callable: Checked[Callable] = run match { - case Run.CommandLineTool(clt) => clt.buildTaskDefinition(validator, expressionLib) - case Run.Workflow(wf) => wf.womDefinition(validator, expressionLib) - case Run.ExpressionTool(et) => et.buildTaskDefinition(validator, expressionLib) - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - - val callNodeBuilder = new CallNode.CallNodeBuilder() - - /* - * Method used to fold over the list of inputs declared by this step. - * Note that because we work on saladed CWL, all ids are fully qualified at this point (e.g: file:///path/to/file/three_step.cwl#cgrep/pattern - * The goal of this method is two fold (pardon the pun): - * 1) link each input of the step to an output port (which at this point can be from a different step or from a workflow input) - * 2) accumulate the nodes created along the way to achieve 1) - */ - def foldStepInput(currentFold: Checked[WorkflowStepInputFold], workflowStepInput: WorkflowStepInput): Checked[WorkflowStepInputFold] = currentFold flatMap { - fold => - /* - * Try to find in the given set an output port named stepOutputId in a call node named stepId - * This is useful when we've determined that the input points to an output of a different step and we want - * to get the corresponding output port. - */ - def findThisInputInSet(set: Set[GraphNode], stepId: String, stepOutputId: String): Checked[OutputPort] = { - for { - // We only care for outputPorts of call nodes or scatter nodes - call <- set.collectFirst { - case callNode: CallNode if callNode.localName == stepId => callNode - case scatterNode: ScatterNode if scatterNode.innerGraph.calls.exists(_.localName == stepId) => scatterNode - }. - toRight(NonEmptyList.one(s"stepId $stepId not found in known Nodes $set")) - output <- call.outputPorts.find(_.internalName == stepOutputId). - toRight(NonEmptyList.one(s"step output id $stepOutputId not found in ${call.outputPorts}")) - } yield output - } - - /* - * Build a wom node for the given step and return the newly created nodes - * This is useful when we've determined that the input belongs to an upstream step that we haven't covered yet - */ - def buildUpstreamNodes(upstreamStepId: String, accumulatedNodes: Set[GraphNode]): Checked[Set[GraphNode]] = - // Find the step corresponding to this upstreamStepId in the set of all the steps of this workflow - for { - step <- workflow.steps.find { step => cwl.FullyQualifiedName(step.id).id == upstreamStepId }. - toRight(NonEmptyList.one(s"no step of id $upstreamStepId found in ${workflow.steps.map(_.id).toList}")) - call <- step.callWithInputs(typeMap, workflow, accumulatedNodes, workflowInputs, validator, expressionLib) - } yield call - - def fromWorkflowInput(inputName: String): Checked[Map[String, OutputPort]] = { - // Try to find it in the workflow inputs map, if we can't it's an error - workflowInputs.collectFirst { - case (inputId, port) if inputName == inputId => Map(inputId -> port).asRight[NonEmptyList[String]] - } getOrElse s"Can't find workflow input for $inputName".invalidNelCheck[Map[String, OutputPort]] - } - - def fromStepOutput(stepId: String, stepOutputId: String, accumulatedNodes: Set[GraphNode]): Checked[(Map[String, OutputPort], Set[GraphNode])] = { - // First check if we've already built the WOM node for this step, and if so return the associated output port - findThisInputInSet(accumulatedNodes, stepId, stepOutputId).map(outputPort => (Map(s"$stepId/$stepOutputId" -> outputPort), accumulatedNodes)) - .orElse { - // Otherwise build the upstream nodes and look again in those newly created nodes - for { - newNodes <- buildUpstreamNodes(stepId, accumulatedNodes) - sourceMappings <- findThisInputInSet(newNodes, stepId, stepOutputId).map(outputPort => Map(s"$stepId/$stepOutputId" -> outputPort)) - } yield (sourceMappings, newNodes ++ accumulatedNodes) - } - } - - lazy val workflowStepInputId = cwl.FullyQualifiedName(workflowStepInput.id).id - - def updateFold(sourceMappings: Map[String, OutputPort], newNodes: Set[GraphNode]): Checked[WorkflowStepInputFold] = { - val typeExpectedByRunInput: Option[cwl.MyriadInputType] = typedRunInputs.get(workflowStepInputId).flatten - - val isThisStepScattered = isStepScattered(workflowStepInputId) - - workflowStepInput.toMergeNode(sourceMappings, expressionLib, typeExpectedByRunInput, isThisStepScattered, allRequirements.schemaDefRequirement) match { - // If the input needs a merge node, build it and add it to the input fold - case Some(mergeNode) => - mergeNode.toEither.map({ node => - fold |+| WorkflowStepInputFold( - mergeNodes = Map(workflowStepInput -> node), - generatedNodes = newNodes - ) - }) - case None => (fold |+| WorkflowStepInputFold(generatedNodes = newNodes)).validNelCheck - } - } - - /* - * We intend to validate that all of these sources point to a WOM Outputport that we know about. - * - * If we don't know about them, we find upstream nodes and build them (see "buildUpstreamNodes"). - */ - val baseCase = (Map.empty[String, OutputPort], fold.generatedNodes).asRight[NonEmptyList[String]] - val inputMappingsAndGraphNodes: Checked[(Map[String, OutputPort], Set[GraphNode])] = - workflowStepInput.sources.foldLeft(baseCase) { - case (Right((sourceMappings, graphNodes)), inputSource) => - /* - * Parse the inputSource (what this input is pointing to) - * 2 cases: - * - points to a workflow input - * - points to an upstream step - */ - cwl.FullyQualifiedName(inputSource) match { - // The source points to a workflow input, which means it should be in the workflowInputs map - case FileAndId(_, _, inputId) => fromWorkflowInput(inputId).map(newMap => (sourceMappings ++ newMap, graphNodes)) - // The source points to an output from a different step - case FileStepAndId(_, _, stepId, stepOutputId) => fromStepOutput(stepId, stepOutputId, graphNodes).map({ case (newMap, newNodes) => (sourceMappings ++ newMap, newNodes) }) - } - case (other, _) => other - } - - inputMappingsAndGraphNodes.flatMap((updateFold _).tupled) - } - - /* - * Folds over input definitions and build an InputDefinitionFold - */ - def foldInputDefinition(pointerNode: Map[String, GraphNodeWithSingleOutputPort]) - (inputDefinition: InputDefinition): ErrorOr[InputDefinitionFold] = { - inputDefinition match { - case _ if pointerNode.contains(inputDefinition.name) => - val expressionNode = pointerNode(inputDefinition.name) - InputDefinitionFold( - mappings = List(inputDefinition -> expressionNode.inputDefinitionPointer), - callInputPorts = Set(callNodeBuilder.makeInputPort(inputDefinition, expressionNode.singleOutputPort)) - ).validNel - - // No expression node mapping, use the default - case withDefault @ OverridableInputDefinitionWithDefault(_, _, expression, _, _) => - InputDefinitionFold( - mappings = List(withDefault -> Coproduct[InputDefinitionPointer](expression)) - ).validNel - - // Required input without default value and without mapping, this is a validation error - case RequiredInputDefinition(requiredName, _, _, _) => - s"Input ${requiredName.value} is required and is not bound to any value".invalidNel - - // Optional input without mapping, defaults to empty value - case optional: OptionalInputDefinition => - InputDefinitionFold( - mappings = List(optional -> Coproduct[InputDefinitionPointer](optional.womType.none: WomValue)) - ).validNel - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - } - - /* - If the step is being scattered over, then merge nodes can't directly be referenced because they will be outside the scatter graph. - For inputs that are being scattered over, a scatter variable has already been created, but for the others we need - an OGIN to link the merge node to the inner scatter graph. - */ - def buildOGINs(mergeNodes: Map[WorkflowStepInput, ExpressionNode], - scatterVariables: Map[WorkflowStepInput, ScatterVariableNode]): Map[WorkflowStepInput, OuterGraphInputNode] = if (isScattered) { - mergeNodes - .collect({ - case (input, mergeNode) if !scatterVariables.contains(input) => - val ogin = OuterGraphInputNode( - WomIdentifier(input.parsedId).combine("OGIN"), - mergeNode.singleOutputPort, - preserveScatterIndex = false - ) - input -> ogin - }) - } else Map.empty - - /* - * For inputs that have a valueFrom field, create an ExpressionNode responsible for evaluating the expression. - * Note that this expression might need access to the other input values, so make each expression node depend on all other - * inputs. - */ - def buildStepInputValueFromNodes(sharedInputNodes: Map[WorkflowStepInput, GraphNodeWithSingleOutputPort]): Checked[Map[String, ExpressionNode]] = { - // Add new information to the typeMap from the shard input nodes. - lazy val updatedTypeMap = sharedInputNodes.map({ - // If the input node is a scatter variable, make sure the type is the item type, not the array type, as the expression node - // will operate on shards not on the whole scattered array. - case (stepInput, scatter: ScatterVariableNode) => stepInput.parsedId -> scatter.womType - case (stepInput, node) => stepInput.parsedId -> node.singleOutputPort.womType - }) ++ typeMap - - // Go over each step input and create an expression node for those which have a valueFrom - in.toList.collect({ - case stepInput @ WorkflowStepInput(_, _, _, _, Some(valueFrom)) => - // Transform the shared inputs map into a usable map to create the expression. - lazy val sharedInputMap: Map[String, OutputPort] = sharedInputNodes.map({ - case (siblingStepInput, graphNode) => siblingStepInput.parsedId -> graphNode.singleOutputPort - }) - val typeExpectedByRunInput: Option[cwl.MyriadInputType] = typedRunInputs.get(stepInput.parsedId).flatten - val isThisStepScattered = isStepScattered(stepInput.parsedId) - - stepInput.toExpressionNode(valueFrom, typeExpectedByRunInput, isThisStepScattered, sharedInputMap, updatedTypeMap, expressionLib, allRequirements.schemaDefRequirement).map(stepInput.parsedId -> _) - }) - .sequence[ErrorOr, (String, ExpressionNode)] - .toEither - .map(_.toMap) - } - - //inputs base case consist of the nodes we already know about - val baseCase = WorkflowStepInputFold(generatedNodes = knownNodes).asRight[NonEmptyList[String]] - - // WorkflowStepInputFold contains the mappings from step input to ExpressionNode as well as all created nodes - val stepInputFoldCheck: Checked[WorkflowStepInputFold] = in.foldLeft(baseCase)(foldStepInput) - - /* - * This (big) flatMap builds nodes from top to bottom in the diagram above. - * If necessary, the scatter node is built last as it wraps some of the other nodes. - */ - for { - /* ************************************ */ - /* ************ Merge Nodes *********** */ - /* ************************************ */ - // Build merge nodes and recursively generates other call nodes that we haven't seen so far - stepInputFold <- stepInputFoldCheck - // Extract the merge nodes from the fold - mergeNodes = stepInputFold.mergeNodes - - /* ************************************ */ - /* ****** Scatter Variable Nodes ****** */ - /* ************************************ */ - scatterVariableNodes <- ScatterLogic.buildScatterVariableNodes(scatter, mergeNodes, unqualifiedStepId.localName.value) - - /* ************************************ */ - /* *************** OGINS ************** */ - /* ************************************ */ - ogins = buildOGINs(mergeNodes, scatterVariableNodes) - - /* ************************************ */ - /* ********* Expression Nodes ********* */ - /* ************************************ */ - // Aggregate the generated nodes so far. This map will be used to generate expression nodes, so the order of aggregation matters: - // scatter variables and ogins take precedence over merge nodes (see diagram) - aggregatedMapForValueFromNodes = mergeNodes ++ scatterVariableNodes ++ ogins - // Build expression nodes for inputs that have a valueFrom field - stepInputValueFromNodes <- buildStepInputValueFromNodes(aggregatedMapForValueFromNodes) - - /* ************************************ */ - /* ************* Call Node ************ */ - /* ************************************ */ - // Get the callable object for this step - checkedCallable <- callable - // Aggregate again by adding generated expression nodes. Again order matters here, expression nodes override other nodes. - aggregatedMapForInputDefinitions = aggregatedMapForValueFromNodes.asIdentifierMap ++ stepInputValueFromNodes - // Assign each of the callable's input definition to an output port from the pointer map - inputDefinitionFold <- checkedCallable.inputs.foldMap(foldInputDefinition(aggregatedMapForInputDefinitions)).toEither - // Build the call node - callAndNodes = callNodeBuilder.build(unqualifiedStepId, checkedCallable, inputDefinitionFold, Set.empty, None) - // Depending on whether the step is being scattered, invoke the scatter node builder or not - - /* ************************************ */ - /* ************ Scatter Node ********** */ - /* ************************************ */ - scatterNodeOrExposedNodes <- if (isScattered) { - ScatterLogic.buildScatterNode( - callAndNodes, - NonEmptyList.fromListUnsafe(scatterVariableNodes.values.toList), - ogins.values.toSet, - stepInputValueFromNodes.values.toSet, - scatterMethod).map(Set(_)) - } else { - // If there's no scatter node then we need to return the expression nodes and the call node explicitly - // as they won't be contained in the scatter inner graph - (stepInputValueFromNodes.values.toSet + callAndNodes.node).validNelCheck - } - - /* - * Return all the nodes that need to be made available to the workflow graph: - * knownNodes: this method is used to fold over steps so we don't want to forget to accumulate known nodes - * mergeNodes: they're always outside of the scatter so always return them - * generatedNodes: nodes generated recursively to build this node - * scatterNodeOrExposedNodes: see explanation above - */ - allNodes = knownNodes ++ mergeNodes.values.toSet ++ stepInputFold.generatedNodes ++ scatterNodeOrExposedNodes - } yield allNodes - } - } -} - -/** - * @see WorkflowstepOutput - */ -case class WorkflowStepOutput(id: String) - -object WorkflowStep { - - // A monoid can't be derived automatically for this class because it contains a Map[String, ExpressionNode], - // and there's no monoid defined over ExpressionNode - implicit val workflowStepInputFoldMonoid: Monoid[WorkflowStepInputFold] = new Monoid[WorkflowStepInputFold] { - override def empty: WorkflowStepInputFold = WorkflowStepInputFold() - override def combine(x: WorkflowStepInputFold, y: WorkflowStepInputFold): WorkflowStepInputFold = { - WorkflowStepInputFold( - mergeNodes = x.mergeNodes ++ y.mergeNodes, - generatedNodes = x.generatedNodes ++ y.generatedNodes - ) - } - } - - private [cwl] case class WorkflowStepInputFold(mergeNodes: Map[WorkflowStepInput, ExpressionNode] = Map.empty, - generatedNodes: Set[GraphNode] = Set.empty) - - /** - * Maps input variable (to be scattered over) to their scatter variable node - */ - type ScatterMappings = Map[ExpressionNode, ScatterVariableNode] - - val emptyOutputs: WorkflowStepOutputType = Array.empty - - type Run = - String :+: - CommandLineTool :+: - ExpressionTool :+: - Workflow :+: - CNil - - object Run { - object String { def unapply(run: Run): Option[String] = run.select[String] } - object Workflow { def unapply(run: Run): Option[Workflow] = run.select[Workflow] } - object CommandLineTool { def unapply(run: Run): Option[CommandLineTool] = run.select[CommandLineTool] } - object ExpressionTool { def unapply(run: Run): Option[ExpressionTool] = run.select[ExpressionTool] } - } - - type WorkflowStepOutputInnerType = String :+: WorkflowStepOutput :+: CNil - type WorkflowStepOutputType = Array[WorkflowStepOutputInnerType] -} diff --git a/cwl/src/main/scala/cwl/WorkflowStepInput.scala b/cwl/src/main/scala/cwl/WorkflowStepInput.scala deleted file mode 100644 index a3d028dbdd6..00000000000 --- a/cwl/src/main/scala/cwl/WorkflowStepInput.scala +++ /dev/null @@ -1,155 +0,0 @@ -package cwl - -import cats.data.NonEmptyList -import cats.syntax.either._ -import common.Checked -import common.validation.Checked._ -import common.validation.ErrorOr.ErrorOr -import cwl.InputParameter.DefaultToWomValuePoly -import cwl.LinkMergeMethod.LinkMergeMethod -import cwl.WorkflowStepInput.InputSource -import cwl.command.ParentName -import shapeless.{:+:, CNil} -import wom.expression.ValueAsAnExpression -import wom.graph.GraphNodePort.OutputPort -import wom.graph.WomIdentifier -import wom.graph.expression.{AnonymousExpressionNode, ExposedExpressionNode, ExpressionNode, PlainAnonymousExpressionNode} -import wom.types._ - -case class WorkflowStepInput( - id: String, - source: Option[InputSource] = None, - linkMerge: Option[LinkMergeMethod] = None, - default: Option[CwlAny] = None, - valueFrom: Option[StringOrExpression] = None) { - - def parsedId(implicit parentName: ParentName) = FullyQualifiedName(id).id - - def toExpressionNode(valueFromExpression: StringOrExpression, - runInputExpectedType: Option[cwl.MyriadInputType], - isScattered: Boolean, - sourceMappings:Map[String, OutputPort], - outputTypeMap: Map[String, WomType], - expressionLib: ExpressionLib, - schemaDefRequirement: SchemaDefRequirement - )(implicit parentName: ParentName): ErrorOr[ExpressionNode] = { - val inputs = sourceMappings.keySet - val upstreamMergeType = outputTypeMap.get(parsedId) - - (for { - // we may have several sources, we make sure to have a type common to all of them. - // In the case where there's no input source, we currently wrap the valueFrom value in a WomString (see WorkflowStepInputExpression) - inputType <- WorkflowStepInput.determineValueFromType(upstreamMergeType, runInputExpectedType, isScattered, schemaDefRequirement) - womExpression = WorkflowStepInputExpression(parsedId, valueFromExpression, inputType, inputs, expressionLib) - identifier = WomIdentifier(id).combine("expression") - ret <- ExposedExpressionNode.fromInputMapping(identifier, womExpression, inputType, sourceMappings).toEither - } yield ret).toValidated - } - - /** - * - * @param sourceMappings The outputports to which this source refers - * @param matchingRunInputType This input matches an input declared in the workflowstep's "run". This is that step's declared type - * @return - */ - def toMergeNode(sourceMappings: Map[String, OutputPort], - expressionLib: ExpressionLib, - matchingRunInputType: Option[MyriadInputType], - isScattered: Boolean, - schemaDefRequirement: SchemaDefRequirement - ): Option[ErrorOr[ExpressionNode]] = { - - val identifier = WomIdentifier(id).combine("merge") - val mapType = sourceMappings.map({ case (k, v) => k -> v.womType }) - - val maybeMatchingRunInputWomType: Option[WomType] = matchingRunInputType.map(_.fold(MyriadInputTypeToWomType).apply(schemaDefRequirement)) - - def makeNode(head: (String, OutputPort), tail: List[(String, OutputPort)]) = for { - inputType <- determineMergeType(mapType, maybeMatchingRunInputWomType) - womExpression = WorkflowStepInputMergeExpression(this, inputType, NonEmptyList.of(head, tail: _*), expressionLib) - node <- AnonymousExpressionNode.fromInputMapping(identifier, womExpression, sourceMappings, PlainAnonymousExpressionNode.apply).toEither - } yield node - - val matchingRunInputWomType: WomType = maybeMatchingRunInputWomType.getOrElse(WomAnyType) - lazy val defaultValue = default.map { _.fold(DefaultToWomValuePoly).apply(matchingRunInputWomType) } - - sourceMappings.toList match { - case head :: tail => Option(makeNode(head, tail).toValidated) - case Nil => - defaultValue map { _ map { d => - PlainAnonymousExpressionNode(identifier, ValueAsAnExpression(d), matchingRunInputWomType, Map.empty) - } } - } - } - - def determineMergeType(sources: Map[String, WomType], expectedTypeAsWom: Option[WomType]): Checked[WomType] = { - WorkflowStepInput.determineMergeType(sources, linkMerge, expectedTypeAsWom, default.isDefined) - } - - lazy val sources: List[String] = source.toList.flatMap(_.fold(StringOrStringArrayToStringList)) - - lazy val effectiveLinkMerge: LinkMergeMethod = linkMerge.getOrElse(LinkMergeMethod.MergeNested) -} - -object WorkflowStepInput { - type InputSource = String :+: Array[String] :+: CNil - - implicit class EnhancedStepInputMap[A](val map: Map[WorkflowStepInput, A]) extends AnyVal { - def asIdentifierMap(implicit parentName: ParentName): Map[String, A] = { - map.map({ case (stepInput, value) => stepInput.parsedId -> value }) - } - } - - def determineValueFromType(mergedSourcesType: Option[WomType], - expectedType: Option[MyriadInputType], - isScattered: Boolean, - schemaDefRequirement: SchemaDefRequirement): Checked[WomType] = { - val expectedTypeAsWom: Option[WomType] = expectedType.map(_.fold(MyriadInputTypeToWomType).apply(schemaDefRequirement)) - - expectedTypeAsWom.getOrElse(WomStringType).asRight - } - - def determineMergeType(sources: Map[String, WomType], - linkMerge: Option[LinkMergeMethod], - expectedTypeAsWom: Option[WomType], - hasDefault: Boolean): Checked[WomType] = { - - (sources.toList, expectedTypeAsWom, linkMerge) match { - // If there is a single source and no explicit merge method, use the type of the source (unpacked if it's optional - // and there's a default since the default would be used if there's no supplied value). - case (List((_, WomOptionalType(sourceType))), _, None) if hasDefault => sourceType.asRight - case (List((_, opt @ WomOptionalType(_))), _, None) => opt.asRight - case (List((_, sourceType)), _, None) => sourceType.asRight - // If there is a single source and merge nested is specified, wrap it into an array - case (List((_, sourceType)), _, Some(LinkMergeMethod.MergeNested)) => WomArrayType(sourceType).asRight - // If there are multiple sources and either no merge method or merge nested, find the closest common type. - // TODO: This is technically not enough, cwltool supports sources with totally different types, creating an array with multiple types - // Maybe WomCoproduct can help - case (_, _, Some(LinkMergeMethod.MergeNested) | None) => WomArrayType(WomType.homogeneousTypeFromTypes(sources.values)).asRight - - //If sink parameter is an array and merge_flattened is used, must validate input & output types are equivalent before proceeding - case (_, Some(arrayType @ WomArrayType(itemType)), Some(LinkMergeMethod.MergeFlattened)) if typesToItemMatch(sources.values, itemType) => arrayType.asRight - // If sink parameter is not an array and merge flattened is used, validate that the sources types matche the sink type - case (_, Some(targetType), Some(LinkMergeMethod.MergeFlattened)) if typesToItemMatch(sources.values, targetType) => WomArrayType(targetType).asRight - // If the types are not compatible, fail - case (_, Some(targetType), Some(LinkMergeMethod.MergeFlattened)) => - s"could not verify that types $sources and the items type of the run's InputArraySchema $targetType were compatible".invalidNelCheck - - //We don't have type information from the run input so we gather up the sources and try to determine a common type amongst them. - case _ => WomType.homogeneousTypeFromTypes(sources.values).asRight - } - } - - def typesToItemMatch(lst: Iterable[WomType], target: WomType): Boolean = { - val effectiveInputType = WomType.homogeneousTypeFromTypes(lst) - - typeToItemMatch(effectiveInputType, target) - } - - def typeToItemMatch(upstream: WomType, downstream: WomType): Boolean = { - upstream match { - case WomType.RecursiveType(innerType) => typeToItemMatch(innerType, downstream) - case other => other == downstream - } - } -} diff --git a/cwl/src/main/scala/cwl/WorkflowStepInputExpression.scala b/cwl/src/main/scala/cwl/WorkflowStepInputExpression.scala deleted file mode 100644 index 476d7f44ef5..00000000000 --- a/cwl/src/main/scala/cwl/WorkflowStepInputExpression.scala +++ /dev/null @@ -1,51 +0,0 @@ -package cwl - -import cats.syntax.validated._ -import wom.expression.{FileEvaluation, IoFunctionSet} -import wom.types._ -import wom.values._ - -final case class WorkflowStepInputExpression(inputName: String, - valueFrom: StringOrExpression, - override val cwlExpressionType: WomType, - inputs: Set[String], - override val expressionLib: ExpressionLib) extends CwlWomExpression { - - override def sourceString = inputName - - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet) = { - valueFrom match { - // If valueFrom is a constant string value, use this as the value for this input parameter. - // TODO: need to handle case where this is a parameter reference, it currently looks like a String to us! - case StringOrExpression.String(value) => WomString(value).validNel - - /* - * If valueFrom is a parameter reference or expression, it must be evaluated to yield the actual value to be assiged to the input field. - * - * The self value of in the parameter reference or expression must be the value of the parameter(s) specified in the source field, - * or null if there is no source field. - * - * The value of inputs in the parameter reference or expression must be the input object to the workflow step after - * assigning the source values, applying default, and then scattering. The order of evaluating valueFrom among step - * input parameters is undefined and the result of evaluating valueFrom on a parameter must not be visible to - * evaluation of valueFrom on other parameters. - */ - case StringOrExpression.Expression(expression) => - //used to determine the value of "self" as expected by CWL Spec - def selfValue = inputValues.get(inputName) match { - case Some(value) => value - case None => WomOptionalValue(WomNothingType, None) - } - - val parameterContext = ParameterContext(ioFunctionSet, expressionLib, inputValues, selfValue) - - expression.fold(EvaluateExpression).apply(parameterContext) - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - } - - //this is input, so not producing any output files - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType) = - Set.empty[FileEvaluation].validNel -} - diff --git a/cwl/src/main/scala/cwl/WorkflowStepInputMergeExpression.scala b/cwl/src/main/scala/cwl/WorkflowStepInputMergeExpression.scala deleted file mode 100644 index bc7d4f43d75..00000000000 --- a/cwl/src/main/scala/cwl/WorkflowStepInputMergeExpression.scala +++ /dev/null @@ -1,84 +0,0 @@ -package cwl - -import cats.data.NonEmptyList -import cats.syntax.either._ -import cats.syntax.option._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import common.Checked -import common.validation.ErrorOr.ErrorOr -import cwl.InputParameter.DefaultToWomValuePoly -import wom.expression.{FileEvaluation, IoFunctionSet} -import wom.graph.GraphNodePort.OutputPort -import wom.types.WomType -import wom.values.{WomArray, WomFile, WomOptionalValue, WomValue} - -final case class WorkflowStepInputMergeExpression(input: WorkflowStepInput, - cwlExpressionType: WomType, - stepInputMappingHead: NonEmptyList[(String, OutputPort)], - override val expressionLib: ExpressionLib) extends CwlWomExpression { - - private val allStepInputMappings = stepInputMappingHead.toList - private val allStepInputSources = allStepInputMappings.map(_._1) - - override def sourceString: String = s"${input.id}-Merge-Expression" - override def inputs: Set[String] = allStepInputSources.toSet - - override def evaluateValue(inputValues: Map[String, WomValue], ioFunctionSet: IoFunctionSet): ErrorOr[WomValue] = { - def lookupValue(key: String): ErrorOr[WomValue] = - inputValues. - get(key). - toValidNel(s"source value $key not found in input values ${inputValues.mkString("\n")} and no default value provided. Graph Inputs were ${allStepInputSources.mkString("\n")}") - - def validateSources(sources: List[String]): ErrorOr[List[WomValue]] = - sources. - traverse(lookupValue) - - def isEmptyOptionalValue(womValue: WomValue): Boolean = womValue match { - case WomOptionalValue(_, None) => true - case _ => false - } - - (allStepInputSources, input.effectiveLinkMerge, input.default) match { - // When we have a single source but no value was provided for it and there's a default. - case (List(source), LinkMergeMethod.MergeNested, Some(default)) if isEmptyOptionalValue(inputValues(source)) => - default.fold(DefaultToWomValuePoly).apply(cwlExpressionType) - - case (List(source), LinkMergeMethod.MergeNested, _) => lookupValue(source) - - //When we have several sources, validate they are all present and provide them as a nested array - case (sources, LinkMergeMethod.MergeNested, _) => validateSources(sources).map(WomArray.apply) - - case (sources, LinkMergeMethod.MergeFlattened, _) => - val validatedSourceValues: Checked[List[WomValue]] = - validateSources(sources).toEither - - def flatten: WomValue => List[WomValue] = { - case WomArray(_, value) => value.toList - case WomOptionalValue(_, Some(value)) => flatten(value) - case other => List(other) - } - - //This is the meat of "merge_flattened," where we find arrays and concatenate them to form one array - val flattenedValidatedSourceValues: Checked[List[WomValue]] = validatedSourceValues.map(_.flatMap(flatten)) - - flattenedValidatedSourceValues.map(list => WomArray(list)).toValidated - - case (List(id), _, _) => lookupValue(id) - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - } - - override def evaluateFiles(inputTypes: Map[String, WomValue], ioFunctionSet: IoFunctionSet, coerceTo: WomType): ErrorOr[Set[FileEvaluation]] = { - if (allStepInputMappings.size > 1) { - // TODO add MultipleInputFeatureRequirement logic in here - "MultipleInputFeatureRequirement not supported yet".invalidNel - } else { - val (inputName, _) = allStepInputMappings.head - inputTypes(inputName).collectAsSeq({ - case file: WomFile => file - }).toSet[WomFile].map(FileEvaluation.requiredFile).validNel - } - } -} diff --git a/cwl/src/main/scala/cwl/command/ParentName.scala b/cwl/src/main/scala/cwl/command/ParentName.scala deleted file mode 100644 index 6e34096d4f3..00000000000 --- a/cwl/src/main/scala/cwl/command/ParentName.scala +++ /dev/null @@ -1,10 +0,0 @@ -package cwl.command - -object ParentName { - def empty: ParentName = ParentName(None) - def apply(id: String): ParentName = ParentName(id.split("#").tail.headOption) -} - -case class ParentName(value: Option[String]) { - def stripParent(in: String) = value.map(v => in.stripPrefix(s"$v/")).getOrElse(in) -} diff --git a/cwl/src/main/scala/cwl/command/StringCommandPart.scala b/cwl/src/main/scala/cwl/command/StringCommandPart.scala deleted file mode 100644 index 9ec668231fb..00000000000 --- a/cwl/src/main/scala/cwl/command/StringCommandPart.scala +++ /dev/null @@ -1,18 +0,0 @@ -package cwl.command - -import cats.syntax.validated._ -import common.validation.ErrorOr.ErrorOr -import wom._ -import wom.callable.RuntimeEnvironment -import wom.expression.IoFunctionSet -import wom.graph.LocalName -import wom.values._ - -case class StringCommandPart(literal: String) extends CommandPart { - override def instantiate(inputsMap: Map[LocalName, WomValue], - functions: IoFunctionSet, - valueMapper: (WomValue) => WomValue, - runtimeEnvironment: RuntimeEnvironment): ErrorOr[List[InstantiatedCommand]] = - // TODO CWL shellquotes by default, but this shellquotes everything. Only shellquote what should be shellquoted. - List(InstantiatedCommand(literal.shellQuote)).validNel -} diff --git a/cwl/src/main/scala/cwl/internal/CommandPartSortingAlgorithm.scala b/cwl/src/main/scala/cwl/internal/CommandPartSortingAlgorithm.scala deleted file mode 100644 index eb6ed0410ce..00000000000 --- a/cwl/src/main/scala/cwl/internal/CommandPartSortingAlgorithm.scala +++ /dev/null @@ -1,71 +0,0 @@ -package cwl.internal - -import cats.data.Kleisli._ -import cats.data.ReaderT -import cats.data.Validated._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import cwl.CommandLineTool._ -import cwl.command.ParentName -import cwl.{ArgumentCommandLineBinding, ArgumentToCommandPart, CommandLineTool, CommandPartExpression, FullyQualifiedName, InputParameter, MyriadInputTypeToSortedCommandParts, MyriadInputTypeToWomType} -import shapeless.Coproduct -import wom.types.WomStringType - -object CommandPartSortingAlgorithm { - def argumentCommandParts(arguments: Option[Array[CommandLineTool.Argument]]): CommandPartExpression[List[SortKeyAndCommandPart]] = - // arguments is an Option[Array[Argument]], the toList.flatten gives a List[Argument] - arguments.toList.flatten - // zip the index because we need it in the sorting key - .zipWithIndex.flatTraverse(argumentToCommandPart.tupled) - - def argumentToCommandPart: (Argument, Int) => CommandPartExpression[List[SortKeyAndCommandPart]] = (argument, index) => ReaderT { - case ((requirementsAndHints, expressionLib, _)) => - val part = argument.fold(ArgumentToCommandPart).apply(requirementsAndHints.hasShellCommandRequirement, expressionLib) - // Get the position from the binding if there is one - val position = argument.select[ArgumentCommandLineBinding].flatMap(_.position) - .map(Coproduct[StringOrInt](_)).getOrElse(CommandLineTool.DefaultPosition) - - // The key consists of the position followed by the index - val sortingKey = CommandBindingSortingKey(List(position, Coproduct[StringOrInt](index))) - - List(SortKeyAndCommandPart(sortingKey, part)).validNel - - } - - def inputBindingsCommandParts(inputs: Array[CommandInputParameter]): CommandPartExpression[List[SortKeyAndCommandPart]] = - inputs.toList.flatTraverse(inputBindingsCommandPart) - - def inputBindingsCommandPart(inputParameter: CommandInputParameter): CommandPartExpression[List[SortKeyAndCommandPart]] = - ReaderT{ case ((hintsAndRequirements, expressionLib, inputValues)) => - val parsedName = FullyQualifiedName(inputParameter.id)(ParentName.empty).id - - val womType = inputParameter.`type`.map(_.fold(MyriadInputTypeToWomType).apply(hintsAndRequirements.schemaDefRequirement)).getOrElse(WomStringType) - - val defaultValue = inputParameter.default.map(_.fold(InputParameter.DefaultToWomValuePoly).apply(womType)) - - inputValues - .collectFirst({ case (inputDefinition, womValue) if inputDefinition.name == parsedName => womValue.validNel }) - .orElse(defaultValue) match { - case Some(Valid(value)) => - // See http://www.commonwl.org/v1.0/CommandLineTool.html#Input_binding - lazy val initialKey = CommandBindingSortingKey.empty - .append(inputParameter.inputBinding, Coproduct[StringOrInt](parsedName)) - - inputParameter.`type`.toList. - flatMap{ - _.fold(MyriadInputTypeToSortedCommandParts). - apply( - inputParameter.inputBinding, - value, - initialKey.asNewKey, - hintsAndRequirements.hasShellCommandRequirement, - expressionLib, - hintsAndRequirements.schemaDefRequirement) - }.validNel - case Some(Invalid(errors)) => Invalid(errors) - case None => s"Could not find an input value for input $parsedName in ${inputValues.prettyString}".invalidNel - } - } - -} diff --git a/cwl/src/main/scala/cwl/internal/CwlEcmaScriptDecoder.scala b/cwl/src/main/scala/cwl/internal/CwlEcmaScriptDecoder.scala deleted file mode 100644 index f7f54d3db1c..00000000000 --- a/cwl/src/main/scala/cwl/internal/CwlEcmaScriptDecoder.scala +++ /dev/null @@ -1,124 +0,0 @@ -package cwl.internal - -import cats.syntax.apply._ -import cats.syntax.traverse._ -import cats.syntax.validated._ -import cats.instances.list._ -import cats.instances.option._ -import common.validation.ErrorOr._ -import common.validation.Validation._ -import cwl.{Directory, File, FileOrDirectory} -import org.mozilla.javascript.{ConsString, NativeArray, NativeObject} -import shapeless.Coproduct -import wom.types.WomNothingType -import wom.values._ - -import scala.jdk.CollectionConverters._ - -class CwlEcmaScriptDecoder { - - def decode(value: AnyRef): ErrorOr[WomValue] = - value match { - case map: NativeObject if map.get("class") == "File" => decodeFile(map.asScala.toMap).flatMap(_.asWomValue) - case map: NativeObject if map.get("class") == "Directory" => decodeDirectory(map.asScala.toMap).flatMap(_.asWomValue) - - case map: NativeObject => decodeMap(map.asScala.toMap) - case array: NativeArray => - val anyList = array.asScala.toList - val anyRefArray = anyList.asInstanceOf[List[AnyRef]] - anyRefArray.traverse(decode).map(WomArray.apply) - - //we represent nulls as this type because Wom doesn't have a "null" value, but it does have a nothing type - //If you wish this to be otherwise please tidy up the Expression interpolator as well - case null => WomOptionalValue(WomNothingType, None).valid - - case string: String => WomString(string).valid - case consString: ConsString => WomString(consString.toString).valid - case int: java.lang.Integer => WomInteger(int).valid - case long: java.lang.Long => WomLong(long).valid - case double: java.lang.Double if double == double.doubleValue.floor && !double.isInfinite => - WomInteger(double.intValue).valid - case double: java.lang.Double => WomFloat(double).valid - case boolean: java.lang.Boolean => WomBoolean(boolean).valid - case unknown => s"While decoding the output $value of the Javascript interpreter, we encountered $unknown and were unable to reify it.".invalidNel - } - - def decodeMap(map: Map[Any, Any]): ErrorOr[WomValue] = { - val realMap: Map[AnyRef, AnyRef] = map.asInstanceOf[Map[AnyRef, AnyRef]] - - val tupleList = realMap.toList.traverse{ - case (k,v) => (k.toString.validNel: ErrorOr[String], decode(v)).tupled - } - val mapWomValues = tupleList.map(_.toMap) - mapWomValues.map(WomObject.apply) - } - - /** - * Called to decode a cwl File or Directory. - */ - def decodeDirectoryOrFile(value: Any): ErrorOr[FileOrDirectory] = { - val invalidValue = s"Not a valid CWL map or directory: $value".invalidNel - - value match { - case map: NativeObject if map.get("class") == "File" => decodeFile(map.asScala.toMap).map(Coproduct[FileOrDirectory](_)) - case map: NativeObject if map.get("class") == "Directory" => decodeDirectory(map.asScala.toMap).map(Coproduct[FileOrDirectory](_)) - case _ => invalidValue - } - } - - /** - * Called to decode an array of cwl File or Directory instances. - */ - def decodeDirectoryOrFiles(value: Any): ErrorOr[Array[FileOrDirectory]] = { - value match { - case na: NativeArray => na.asScala.toList.traverse(decodeDirectoryOrFile).map(_.toArray) - } - } - - /** - * Called to decode a map value using a supplied function. - */ - def decodeMapValue[A](map: Map[Any, Any], key: String, f: Any => A): ErrorOr[Option[A]] = { - map.get(key).traverse(anyRef => validate(f(anyRef))) - } - - /** - * Called to decode an array of files or directories from a map value. - */ - def decodeMapDirectoryOrFiles(map: Map[Any, Any], - key: String): ErrorOr[Option[Array[FileOrDirectory]]] = { - map.get(key).traverse(decodeDirectoryOrFiles) - } - - /** - * Called to decode a cwl File. - */ - def decodeFile(map: Map[Any, Any]): ErrorOr[File] = { - val location = decodeMapValue(map, "location", _.toString) - val path = decodeMapValue(map, "path", _.toString) - val basename = decodeMapValue(map, "basename", _.toString) - val checksum = decodeMapValue(map, "checksum", _.toString) - val size = decodeMapValue(map, "size", _.toString.toDouble.toLong) - val secondaryFiles = decodeMapDirectoryOrFiles(map, "secondaryFiles") - val format = decodeMapValue(map, "format", _.toString) - val contents = decodeMapValue(map, "contents", _.toString) - - (location, path, basename, checksum, size, secondaryFiles, format, contents).mapN( - File(_, _, _, _, _, _, _, _) - ) - } - - /** - * Called to decode a cwl Directory. - */ - def decodeDirectory(map: Map[Any, Any]): ErrorOr[Directory] = { - val location = decodeMapValue(map, "location", _.toString) - val path = decodeMapValue(map, "path", _.toString) - val basename = decodeMapValue(map, "basename", _.toString) - val listing = decodeMapDirectoryOrFiles(map, "listing") - - (location, path, basename, listing).mapN( - Directory(_, _, _, _) - ) - } -} diff --git a/cwl/src/main/scala/cwl/internal/EcmaScriptEncoder.scala b/cwl/src/main/scala/cwl/internal/EcmaScriptEncoder.scala deleted file mode 100644 index e4aed51e215..00000000000 --- a/cwl/src/main/scala/cwl/internal/EcmaScriptEncoder.scala +++ /dev/null @@ -1,123 +0,0 @@ -package cwl.internal - -import cats.data.Validated.Valid -import common.validation.ErrorOr.ErrorOr -import cwl.internal.EcmaScriptUtil.{ECMAScriptVariable, ESArray, ESObject, ESPrimitive} -import cwl.{Directory, File} -import mouse.all._ -import wom.values._ - -/** - * Converts a WomValue into a javascript compatible value. - */ -class EcmaScriptEncoder { - - /** - * Base implementation converts any WomPrimitive (except WomFile) into a javascript compatible value. - * - * Inputs, and returned output must be one of: - * - WomString - * - WomBoolean - * - WomFloat - * - WomInteger - * - WomMap - * - WomArray - * - A "WomNull" equal to WomOptionalValue(WomNothingType, None) - * - * The WomMap keys and values, and WomArray elements must be the one of the above, recursively. - * - * Instances of WomFile are not permitted, and must be already converted to one of the above types. - * - * @param value A WOM value. - * @return The javascript equivalent. - */ - def encode(value: WomValue): ECMAScriptVariable = { - value match { - case file: WomFile => encodeFileOrDirectory(file) - case WomOptionalValue(_, None) => ESPrimitive(null) - case WomOptionalValue(_, Some(innerValue)) => encode(innerValue) - case WomString(string) => string |> ESPrimitive - case WomInteger(int) => Int.box(int) |> ESPrimitive - case WomLong(long) => Long.box(long) |> ESPrimitive - case WomFloat(double) => Double.box(double) |> ESPrimitive - case WomBoolean(boolean) => Boolean.box(boolean) |> ESPrimitive - case WomArray(_, array) => array.toList.map(encode).toArray |> ESArray - case WomMap(_, map) => map.map{ - case (mapKey, mapValue) => (encodeString(mapKey), encode(mapValue)) - } |> ESObject - case objectLike: WomObjectLike => objectLike.values.map{ - case (key, innerValue) => (key, encode(innerValue)) - } |> ESObject - case WomCoproductValue(_, womValue) => encode(womValue) - case WomEnumerationValue(_, womValue) => womValue |> ESPrimitive - case _ => throw new RuntimeException(s"$getClass is unable to encode value: $value") - } - } - - def encodeString(value: WomValue): String = { - encode(value) match { - case ESPrimitive(string: String) => string - - //In the case of a non-string, we evaluate a small snippet of Ecma script meant to coerce the object to a string - // http://2ality.com/2012/03/converting-to-string.html - case _ => - val jsString: ErrorOr[WomValue] = EcmaScriptUtil.evalStructish(""""" + other""","other" -> value, encoder = this) - jsString match { - case Valid(WomString(string)) => string - case unexpected => throw new RuntimeException(s"Expected to convert '$value' to a String but ended up with '$unexpected'") - } - } - } - - /** - * Encodes a sequence of wom file or directory values. - */ - def encodeFileOrDirectories(values: Seq[WomFile]): ESArray = { - ESArray(values.toList.map(encodeFileOrDirectory).toArray) - } - - /** - * Encodes a wom file or directory value. - */ - def encodeFileOrDirectory(value: WomFile): ECMAScriptVariable = { - value match { - case directory: WomUnlistedDirectory => encodeDirectory(WomMaybeListedDirectory(directory.value)) - case file: WomSingleFile => encodeFile(WomMaybePopulatedFile(file.value)) - case glob: WomGlobFile => encodeFile(WomMaybePopulatedFile(glob.value)) - case directory: WomMaybeListedDirectory => encodeDirectory(directory) - case file: WomMaybePopulatedFile => encodeFile(file) - } - } - - /** - * Encodes a wom file. - */ - def encodeFile(file: WomMaybePopulatedFile): ECMAScriptVariable = - List( - Option("class" -> ESPrimitive("File")), - file.valueOption.map("location" -> ESPrimitive(_)), - file.valueOption.map("path" -> ESPrimitive(_)), - Option("basename" -> (File.basename(file.value) |> ESPrimitive)), - Option("dirname" -> (File.dirname(file.value) |> ESPrimitive)), - Option("nameroot" -> (File.nameroot(file.value) |> ESPrimitive)), - Option("nameext" -> (File.nameext(file.value) |> ESPrimitive)), - file.checksumOption.map("checksum" -> ESPrimitive(_)), - file.sizeOption.map(Long.box).map("size" -> ESPrimitive(_)), - Option("secondaryFiles" -> encodeFileOrDirectories(file.secondaryFiles)), - file.formatOption.map("format" -> ESPrimitive(_)), - file.contentsOption.map("contents" -> ESPrimitive(_)) - ).flatten.toMap |> ESObject - - /** - * Encodes a wom directory. - */ - def encodeDirectory(directory: WomMaybeListedDirectory): ESObject = { - List( - Option("class" -> ESPrimitive("Directory")), - directory.valueOption.map("location" -> ESPrimitive(_)), - Option(directory.value).map("path" -> ESPrimitive(_)), - Option("basename" -> ESPrimitive(Directory.basename(directory.value))), - directory.listingOption.map(encodeFileOrDirectories).map("listing" -> _) - ).flatten.toMap |> ESObject - } -} diff --git a/cwl/src/main/scala/cwl/internal/EcmaScriptUtil.scala b/cwl/src/main/scala/cwl/internal/EcmaScriptUtil.scala deleted file mode 100644 index d2c5def745f..00000000000 --- a/cwl/src/main/scala/cwl/internal/EcmaScriptUtil.scala +++ /dev/null @@ -1,128 +0,0 @@ -package cwl.internal - -import common.collections.EnhancedCollections._ -import common.validation.ErrorOr._ -import common.validation.Validation._ -import org.mozilla.javascript._ -import wom.values._ - -import scala.concurrent.duration._ -import scala.util.Try - -/** - * This implementation depends on Mozilla Rhino. - * - * Previously we attempted to use Nashorn which is built into the JDK, but we encountered at least 2 situations where - * it didn't work and we found no workarounds to satisfy the use cases. Namely, JSON.stringify of a Map and calling "sort" on an array. - */ -//noinspection VariablePatternShadow -object EcmaScriptUtil { - def writeValue(value: ECMAScriptVariable)(context: Context, scope: Scriptable): AnyRef = - value match { - case ESObject(fields) => - val newObj = context.newObject(scope) - - fields.toList.foreach{ - case (name, value) => - val newerObj = writeValue(value)(context, scope) - ScriptableObject.putProperty(newObj, name, newerObj) - } - newObj - - case ESArray(array) => - val newObj = context.newArray(scope, array.length) - - array.toList.zipWithIndex.foreach { - case (js, index) => - - val newerObj = writeValue(js)(context, scope) - ScriptableObject.putProperty(newObj, index, newerObj) - - } - newObj - - case ESPrimitive(obj) => obj - } - - /** - * Runs ECMAScript as JS1.8 (aka ES5) - * - * Uses RhinoSandbox to reduce chances injected JS wreaks havoc on the JVM. - * - * @see https://en.wikipedia.org/wiki/ECMAScript#Version_correspondence - */ - def evalRaw(expr: String)(block: (Context, Scriptable) => Unit): AnyRef = { - // TODO: Parameterize and update callers to pass in source name, max duration, max instructions, etc.? - // For now, be very liberal with scripts giving 60 seconds of unrestricted CPU usage and unlimited instructions. - val sourceName = "" - val maxDuration: Duration = 60.seconds - val maxInstructionsOption: Option[Int] = None - val strict = true - val languageVersionOption = Option(Context.VERSION_1_8) - - val sandbox = new EnhancedRhinoSandbox(strict, languageVersionOption) - if (maxDuration.isFinite) { - sandbox.setMaxDuration(maxDuration.toMillis.toInt) - } - maxInstructionsOption foreach sandbox.setInstructionLimit - sandbox.setUseSafeStandardObjects(true) - sandbox.setUseSealedScope(true) - - sandbox.eval(sourceName, expr)(block) - } - - sealed trait ECMAScriptVariable - - case class ESObject(fields: Map[String, ECMAScriptVariable]) extends ECMAScriptVariable - case class ESArray(array: Array[ECMAScriptVariable]) extends ECMAScriptVariable { - override def toString: String = s"ESArray(${array.toList})" - } - case class ESPrimitive(anyRef: AnyRef) extends ECMAScriptVariable - - /** - * Evaluates a javascript expression using maps of WOM values. - * - * TODO: Once custom types are supported as WomValue, this custom method won't be required. - * - * @param expr The javascript expression. - * @param rawValues A map filled with WOM values. - * @param mapValues A map of maps filled with WOM values of various types. - * @param encoder Encodes wom values to javascript. - * @param decoder Decodes wom values from javascript. - * @return The result of the expression. - */ - def evalStructish(expr: String, - rawValues: (String, WomValue), - mapValues: Map[String, Map[String, WomValue]] = Map.empty, - encoder: EcmaScriptEncoder, - decoder: CwlEcmaScriptDecoder = new CwlEcmaScriptDecoder): ErrorOr[WomValue] = { - def evaluate = evalRaw(expr) { (context, scope) => - - val (key, value) = rawValues - - val jsValue = encoder.encode(value) - val field = writeValue(jsValue)(context, scope) - ScriptableObject.putProperty(scope, key, field) - - val jsMap = mapValues.safeMapValues{ _.safeMapValues(encoder.encode) } - - jsMap foreach { - case (scopeId, nestedMap) => - - val newObj = context.newObject(scope) - nestedMap.toList foreach { - case (key, value) => - val newerObj = writeValue(value)(context, scope) - ScriptableObject.putProperty(newObj, key, newerObj) - } - ScriptableObject.putProperty(scope, scopeId, newObj) - } - - } - - for { - evaluated <- Try(evaluate).toErrorOr - decoded <- decoder.decode(evaluated) - } yield decoded - } -} diff --git a/cwl/src/main/scala/cwl/internal/EnhancedRhinoSandbox.scala b/cwl/src/main/scala/cwl/internal/EnhancedRhinoSandbox.scala deleted file mode 100644 index 405c04fdaa2..00000000000 --- a/cwl/src/main/scala/cwl/internal/EnhancedRhinoSandbox.scala +++ /dev/null @@ -1,167 +0,0 @@ -package cwl.internal - -import cwl.internal.EnhancedRhinoSandbox._ -import delight.rhinosandox.internal._ -import org.mozilla.javascript._ - -import scala.jdk.CollectionConverters._ -import scala.reflect._ - -/** - * Extends the RhinoSandboxImpl with some fixes and enhancements using java reflection. - * - * @param strict Should evaluation be strict. - * @param languageVersionOption The optional language version to set. - */ -class EnhancedRhinoSandbox(strict: Boolean = true, languageVersionOption: Option[Int] = None) extends RhinoSandboxImpl { - - // Allows easier reflection access to private fields - private lazy val sandboxImpl: RhinoSandboxImpl = this - - /** - * Similar to RhinoSandbox.eval but passes back the context and scope for mutating before evaluation. - * - * With the original RhinoSandbox.eval not sure how to: - * - Create (nested) arrays - * - Create (nested) maps - * - Set JS/ES version - * - * So this uses a copy-port of the original, using reflection to read some of RhinoSandbox's private variables. - * - * Instead of hiding the context and scope as in RhinoSandbox.eval, both are passed back through the block. - * - * TODO: Ask RhinoSandbox if hacks are even needed, and if so contrib back patches so that reflection isn't required. - * - Is there a way to pass nested maps/arrays via a `java.util.Map[String, Object]`, or must we use our `block`? - * - Additionally: can we skip passing `context` to a block as a thread may just call `Context.getCurrentContext()`? - * - * @see https://maxrohde.com/2015/08/06/sandboxing-javascript-in-java-app-link-collection/ - * @see https://github.com/javadelight/delight-rhino-sandbox/blob/9f5a073/src/main/java/delight/rhinosandox/internal/RhinoSandboxImpl.java#L100-L123 - * @see delight.rhinosandox.internal.RhinoSandboxImpl#assertContextFactory() - */ - def eval(sourceName: String, js: String)(block: (Context, Scriptable) => Unit): AnyRef = { - assertContextFactory() - val sandboxImpl_contextFactory = PrivateField(sandboxImpl, "contextFactory").as[ContextFactory] - // RhinoSandbox diff: eval has enterContext inside the try, but Rhino docs say it belongs outside. - // https://www-archive.mozilla.org/rhino/apidocs/org/mozilla/javascript/ContextFactory.html#enterContext%28%29 - val context = sandboxImpl_contextFactory.enterContext - try { - // RhinoSandbox diff: allow setting the language version - languageVersionOption foreach context.setLanguageVersion - assertSafeScope(context) - val sandboxImpl_globalScope = PrivateField(sandboxImpl, "globalScope").as[ScriptableObject] - val sandboxImpl_sealScope = PrivateField(sandboxImpl, "sealScope").as[Boolean] - if (sandboxImpl_sealScope) { - sandboxImpl_globalScope.sealObject() - } - val sandboxImpl_safeScope = PrivateField(sandboxImpl, "safeScope").as[ScriptableObject] - val instanceScope = context.newObject(sandboxImpl_safeScope) - instanceScope.setPrototype(sandboxImpl_safeScope) - instanceScope.setParentScope(null) - - block(context, instanceScope) - - // RhinoSandbox diff: allow strict JS/ES evaluation - // See note at top assertContextFactory as to why we have to put 'use strict'; here. - // All on one line to avoid off-by-one error for javascript error messages that report line numbers. - // Could also pass zero as the line number, but the RhinoSandbox passes hard codes line number one also. - val script = if (strict) s"'use strict';$js" else js - context.evaluateString(instanceScope, script, sourceName, 1, null) - } finally { - Context.exit() - } - } - - /** - * Stricter context factory modified from RhinoSandboxImpl.assertContextFactory(). - * - * ContextFactory.initGlobal() is called within a static synchronized block. - * The globalScope initialized via initSafeStandardObjects instead of initStandardObjects. - * - * The default implementation uses a SafeContext that allows non-strict JS/ES. We would ideally set - * FEATURE_STRICT_MODE to true but that only produces warnings and doesn't return an error. Unfortunately when - * FEATURE_WARNING_AS_ERROR is enabled then non-strict Rhino warnings like "missing ;" throw errors. Instead, - * "'use strict';" is injected before scripts. - */ - override def assertContextFactory(): Unit = { - if (PrivateField(sandboxImpl, "contextFactory").as[ContextFactory] != null) { - return - } - - val _safeContext = new SafeContext - PrivateField(sandboxImpl, "contextFactory") := _safeContext - val _hasExplicitGlobal = ContextFactory.hasExplicitGlobal - val _not = !_hasExplicitGlobal - // RhinoSandbox diff: the global does not like to be initialized twice. Synchronize initialization. - val sandboxImpl_contextFactory = PrivateField(sandboxImpl, "contextFactory").as[SafeContext] - if (_not) initGlobalSynchronized(sandboxImpl_contextFactory) - - val sandboxImpl_instructionLimit = PrivateField(sandboxImpl, "instructionLimit").as[Int] - PrivateField(sandboxImpl_contextFactory, "maxInstructions") := sandboxImpl_instructionLimit - val sandboxImpl_maxDuration = PrivateField(sandboxImpl, "maxDuration").as[Long] - PrivateField(sandboxImpl_contextFactory, "maxRuntimeInMs") := sandboxImpl_maxDuration - // RhinoSandbox diff: assertContextFactory has enterContext inside the try, but Rhino docs say it belongs outside. - // https://www-archive.mozilla.org/rhino/apidocs/org/mozilla/javascript/ContextFactory.html#enterContext%28%29 - val context = sandboxImpl_contextFactory.enterContext - try { - // RhinoSandbox diff: Default globalScope is created via initStandardObjects instead of initSafeStandardObjects. - // initStandardObjects would add the various java packages into the global scope, including `java.io.File`, etc. - PrivateField(sandboxImpl, "globalScope") := context.initSafeStandardObjects(null, false) - val sandboxImpl_inScope = PrivateField(sandboxImpl, "inScope").as[java.util.Map[String, AnyRef]] - val _entrySet = sandboxImpl_inScope.entrySet - val sandboxImpl_globalScope = PrivateField(sandboxImpl, "globalScope").as[ScriptableObject] - for (entry <- _entrySet.asScala) { - sandboxImpl_globalScope.put( - entry.getKey, - sandboxImpl_globalScope, - Context.toObject(entry.getValue, sandboxImpl_globalScope)) - } - val parameters = Array(classOf[String]) - val dealMethod = classOf[RhinoEvalDummy].getMethod("eval", parameters: _*) - val _rhinoEval = new RhinoEval("eval", dealMethod, sandboxImpl_globalScope) - sandboxImpl_globalScope.defineProperty("eval", _rhinoEval, ScriptableObject.DONTENUM) - } finally { - Context.exit() - } - } - -} - -object EnhancedRhinoSandbox { - - /** - * Get or sets a private field. - * - * @param obj The instance to retrieve the field from. - * @param name The name of the field. - * @tparam A The class to retrieve the value from. The field MUST exist on this class, and not a superclass. - */ - final case class PrivateField[A: ClassTag](obj: A, name: String) { - private[this] def field = { - val field = classTag[A].runtimeClass.getDeclaredField(name) - field.setAccessible(true) - field - } - - def as[B]: B = { - field.get(obj).asInstanceOf[B] - } - - def :=(value: Any): Unit = { - field.set(obj, value) - } - } - - /** - * Call ContextFactory.initGlobal in a static synchronized block. - * - * @see [[cwl.internal.EnhancedRhinoSandbox.assertContextFactory]] - */ - private def initGlobalSynchronized(sandboxImpl_contextFactory: ContextFactory) = { - synchronized { - val _hasExplicitGlobal = ContextFactory.hasExplicitGlobal - val _not = !_hasExplicitGlobal - if (_not) ContextFactory.initGlobal(sandboxImpl_contextFactory) - } - } - -} diff --git a/cwl/src/main/scala/cwl/internal/GigabytesToBytes.scala b/cwl/src/main/scala/cwl/internal/GigabytesToBytes.scala deleted file mode 100644 index 7e45aeec5ba..00000000000 --- a/cwl/src/main/scala/cwl/internal/GigabytesToBytes.scala +++ /dev/null @@ -1,26 +0,0 @@ -package cwl.internal - -import shapeless.{Coproduct, Poly1} - -object GigabytesToBytes extends Poly1 { - val toMebibytesMultiplier = Math.pow(2, 20).toLong - - implicit def long = at[Long] { - l => - val value = l * toMebibytesMultiplier - Coproduct[cwl.ResourceRequirementType](value) - } - - implicit def string = at[String] { - s => - //TODO: Scale this by multiplier https://github.com/broadinstitute/cromwell/issues/3382 - Coproduct[cwl.ResourceRequirementType](s) - } - - implicit def expression = at[cwl.Expression] { - e => - //TODO: Scale this by multiplier https://github.com/broadinstitute/cromwell/issues/3382 - Coproduct[cwl.ResourceRequirementType](e) - } -} - diff --git a/cwl/src/main/scala/cwl/model.scala b/cwl/src/main/scala/cwl/model.scala deleted file mode 100644 index 838ee66a98c..00000000000 --- a/cwl/src/main/scala/cwl/model.scala +++ /dev/null @@ -1,310 +0,0 @@ -package cwl - -import cwl.CommandLineTool.{CommandBindingSortingKey, SortKeyAndCommandPart} -import cwl.SchemaDefRequirement.{AsInputEnumSchema, AsInputRecordSchema, SchemaDefTypes} -import cwl.CwlType.CwlType -import cwl.WorkflowStepInput.InputSource -import cwl.command.ParentName -import cwl.internal.GigabytesToBytes -import eu.timepit.refined._ -import eu.timepit.refined.api.Refined -import eu.timepit.refined.string.MatchesRegex -import shapeless.syntax.singleton._ -import shapeless.{:+:, CNil, Coproduct, Poly1, Witness} -import wom.types.WomType -import wom.values.WomValue -import mouse.all._ - -object WorkflowStepInputSource { - object String { - def unapply(arg: InputSource): Option[String] = arg.select[String] - } - object StringArray { - def unapply(arg: InputSource): Option[Array[String]] = arg.select[Array[String]] - } -} - -/** - * Describes a bespoke type. - * - * @param name This field actually does _not_ appear in the v1.0 schema, but it is used anyway in the conformance tests. - * After some consideration it was determined that we should close our eyes and pretend it is in the spec. It - * makes its formal appearance as a required field in v1.1. - */ -case class InputRecordSchema( - name: String, - fields: Option[Array[InputRecordField]] = None, - `type`: W.`"record"`.T = W("record").value, - label: Option[String] = None) { - -} - -case class InputRecordField( - name: String, - `type`: MyriadInputType, - doc: Option[String] = None, - inputBinding: Option[InputCommandLineBinding], - label: Option[String] = None) - -case class InputArraySchema -( - items: MyriadInputType, - `type`: W.`"array"`.T = Witness("array").value, - label: Option[String] = None, - inputBinding: Option[InputCommandLineBinding] = None, - // IAS.secondaryFiles are NOT listed in 1.0 spec, but according to jgentry they will be, maybe - secondaryFiles: Option[SecondaryFiles] = None -) - -trait CommandLineBinding { - def loadContents: Option[Boolean] - def position: Option[Int] - def prefix: Option[String] - def separate: Option[Boolean] - def itemSeparator: Option[String] - def optionalValueFrom: Option[StringOrExpression] - def shellQuote: Option[Boolean] - // separate defaults to true - def effectiveSeparate = separate.getOrElse(true) -} - -object InputCommandLineBinding { - def default = InputCommandLineBinding() -} - -case class InputCommandLineBinding( - loadContents: Option[Boolean] = None, - position: Option[Int] = None, - prefix: Option[String] = None, - separate: Option[Boolean] = None, - itemSeparator: Option[String] = None, - valueFrom: Option[StringOrExpression] = None, - shellQuote: Option[Boolean] = None) extends CommandLineBinding { - override val optionalValueFrom = valueFrom - - def toCommandPart(sortingKey: CommandBindingSortingKey, boundValue: WomValue, hasShellCommandRequirement: Boolean, expressionLib: ExpressionLib) = { - SortKeyAndCommandPart(sortingKey, InputCommandLineBindingCommandPart(this, boundValue)(hasShellCommandRequirement, expressionLib)) - } -} - -// valueFrom is required for command line bindings in the argument section: http://www.commonwl.org/v1.0/CommandLineTool.html#CommandLineBinding -case class ArgumentCommandLineBinding( - valueFrom: StringOrExpression, - loadContents: Option[Boolean] = None, - position: Option[Int] = None, - prefix: Option[String] = None, - separate: Option[Boolean] = None, - itemSeparator: Option[String] = None, - shellQuote: Option[Boolean] = None) extends CommandLineBinding { - override val optionalValueFrom = Option(valueFrom) -} - -case class InputBinding(position: Int, prefix: String) - -object MyriadOutputInnerTypeCacheableString extends Poly1 { - import Case._ - - private def cacheableOutputRecordFieldString(field: OutputRecordField): String = { - val fieldType = field.`type`.fold(MyriadOutputTypeCacheableString) - val lqn = field.name.substring(field.name.lastIndexOf('#') + 1) - s"OutputRecordField($lqn,$fieldType,${field.doc},${field.outputBinding})" - } - - implicit def recordSchema: Aux[OutputRecordSchema, String] = at[OutputRecordSchema] { - s => - val t = s.`type` - s"OutputRecordSchema($t,${s.fields.map(a => "Array(" + a.map(cacheableOutputRecordFieldString).mkString(",") + ")" )},${s.label})" - } - - implicit def arraySchema: Aux[OutputArraySchema, String] = at[OutputArraySchema] { - a => - val is: String = a.items.fold(MyriadOutputTypeCacheableString) - val t = a.`type` - s"OutputArraySchema($is,$t,${a.label},${a.outputBinding})" - } - - implicit def enumSchema: Aux[OutputEnumSchema, String] = at[OutputEnumSchema] { _.toString } - implicit def cwlType: Aux[CwlType, String] = at[CwlType] { _.toString } - implicit def string: Aux[String, String] = at[String] { identity } -} - - -object MyriadOutputTypeCacheableString extends Poly1 { - import Case._ - - implicit def one: Aux[MyriadOutputInnerType, String] = at[MyriadOutputInnerType] { - _.fold(MyriadOutputInnerTypeCacheableString) - } - - implicit def many: Aux[Array[MyriadOutputInnerType], String] = at[Array[MyriadOutputInnerType]] { a => - val strings: Array[String] = a.map(_.fold(MyriadOutputInnerTypeCacheableString)) - "Array(" + strings.mkString(",") + ")" - } -} - -object SecondaryFilesCacheableString extends Poly1 { - import Case._ - - implicit def one: Aux[StringOrExpression, String] = at[StringOrExpression] { - _.toString - } - - implicit def array: Aux[Array[StringOrExpression], String] = at[Array[StringOrExpression]] { - _.mkString("Array(", ",", ")") - } - -} - -case class OutputRecordSchema( - `type`: W.`"record"`.T, - fields: Option[Array[OutputRecordField]], - label: Option[String]) - -case class OutputRecordField( - name: String, - `type`: MyriadOutputType, - doc: Option[String], - outputBinding: Option[CommandOutputBinding]) - -case class OutputArraySchema( - items: MyriadOutputType, - `type`: W.`"array"`.T = Witness("array").value, - label: Option[String] = None, - outputBinding: Option[CommandOutputBinding] = None) - -case class InlineJavascriptRequirement( - `class`: W.`"InlineJavascriptRequirement"`.T = "InlineJavascriptRequirement".narrow, - expressionLib: Option[Array[String]] = None) - -case class SchemaDefRequirement( - types: Array[SchemaDefTypes] = Array.empty, - `class`: W.`"SchemaDefRequirement"`.T = Witness("SchemaDefRequirement").value) { - - def lookupType(tpe: String): Option[WomType] = - lookupCwlType(tpe).flatMap{ - case AsInputRecordSchema(inputRecordSchema: InputRecordSchema) => MyriadInputInnerTypeToWomType.inputRecordSchemaToWomType(inputRecordSchema).apply(this) |> Option.apply - case _ => None - } - - //Currently only InputRecordSchema has a name in the spec, so it is the only thing that can be referenced via string - def lookupCwlType(tpe: String): Option[SchemaDefTypes] = { - - def matchesType(inputEnumSchema: InputEnumSchema): Boolean = { - inputEnumSchema.name.fold(false){name => FileAndId(name)(ParentName.empty).id equalsIgnoreCase FileAndId(tpe)(ParentName.empty).id} - } - - types.toList.flatMap { - case AsInputRecordSchema(inputRecordSchema: InputRecordSchema) if FileAndId(inputRecordSchema.name)(ParentName.empty).id equalsIgnoreCase FileAndId(tpe)(ParentName.empty).id => - List(Coproduct[SchemaDefTypes](inputRecordSchema)) - case AsInputEnumSchema(inputEnumSchema: InputEnumSchema) if matchesType(inputEnumSchema) => - List(Coproduct[SchemaDefTypes](inputEnumSchema)) - case _ => - List() - }.headOption - } -} - -object SchemaDefRequirement { - type SchemaDefTypes = InputRecordSchema :+: InputEnumSchema :+: InputArraySchema :+: CNil - - object AsInputRecordSchema { - def unapply(arg: SchemaDefTypes): Option[InputRecordSchema] = arg.select[InputRecordSchema] - } - - object AsInputEnumSchema { - def unapply(arg: SchemaDefTypes): Option[InputEnumSchema] = arg.select[InputEnumSchema] - } - - object AsInputArraySchema { - def unapply(arg: SchemaDefTypes): Option[InputArraySchema] = arg.select[InputArraySchema] - } -} - -//There is a large potential for regex refinements on these string types -case class DockerRequirement( - `class`: W.`"DockerRequirement"`.T, - dockerPull: Option[String], //TODO Refine to match a docker image regex? - dockerLoad: Option[String], - dockerFile: Option[String], - dockerImport: Option[String], - dockerImageId: Option[String], - dockerOutputDirectory: Option[String] - ) - -case class SoftwareRequirement( - `class`: W.`"SoftwareRequirement"`.T, - packages: Array[SoftwarePackage] = Array.empty - ) - -case class SoftwarePackage( - `package`: String, - version: Option[Array[String]], - specs: Option[Array[String]] // This could be refined to match a regex for IRI. - ) { - type Package = String - type Specs = Array[String] -} - -case class EnvVarRequirement( - `class`: EnvVarRequirement.ClassType = EnvVarRequirement.`class`, - envDef: Array[EnvironmentDef] - ) - -object EnvVarRequirement { - type ClassType = Witness.`"EnvVarRequirement"`.T - - val `class`: ClassType = "EnvVarRequirement".asInstanceOf[ClassType] -} - -case class EnvironmentDef(envName: String, envValue: StringOrExpression) { - type EnvName = String - type EnvValue = String -} - - -case class ShellCommandRequirement(`class`: W.`"ShellCommandRequirement"`.T = "ShellCommandRequirement".narrow) - -case class ResourceRequirement( - `class`: W.`"ResourceRequirement"`.T, - coresMin: Option[ResourceRequirementType], - coresMax: Option[ResourceRequirementType], - ramMin: Option[ResourceRequirementType], - ramMax: Option[ResourceRequirementType], - tmpdirMin: Option[ResourceRequirementType], - tmpdirMax: Option[ResourceRequirementType], - outdirMin: Option[ResourceRequirementType], - outdirMax: Option[ResourceRequirementType]) { - def effectiveCoreMin = coresMin.orElse(coresMax) - def effectiveCoreMax = coresMax.orElse(coresMin) - - def effectiveRamMin = ramMin.orElse(ramMax).map(_.fold(GigabytesToBytes)) - def effectiveRamMax = ramMax.orElse(ramMin).map(_.fold(GigabytesToBytes)) - - def effectiveTmpdirMin = tmpdirMin.orElse(tmpdirMax) - def effectiveTmpdirMax = tmpdirMax.orElse(tmpdirMin) - - def effectiveOutdirMin = outdirMin.orElse(outdirMax) - def effectiveOutdirMax = outdirMax.orElse(outdirMin) -} - -/** - * This promotes DNA Nexus InputResourceRequirement to a first class citizen requirement, which it really isn't. - * Since it's the only one for now it's not a big deal but if more of these pop up we might want to treat custom requirements - * in a different way - */ -case class DnaNexusInputResourceRequirement( - `class`: String Refined MatchesRegex[W.`".*InputResourceRequirement"`.T], - indirMin: Option[Long] - ) - -case class SubworkflowFeatureRequirement( - `class`: W.`"SubworkflowFeatureRequirement"`.T) - -case class ScatterFeatureRequirement( - `class`: W.`"ScatterFeatureRequirement"`.T) - -case class MultipleInputFeatureRequirement( - `class`: W.`"MultipleInputFeatureRequirement"`.T) - -case class StepInputExpressionRequirement( - `class`: W.`"StepInputExpressionRequirement"`.T) diff --git a/cwl/src/main/scala/cwl/ontology/CacheConfiguration.scala b/cwl/src/main/scala/cwl/ontology/CacheConfiguration.scala deleted file mode 100644 index cb82d9db591..00000000000 --- a/cwl/src/main/scala/cwl/ontology/CacheConfiguration.scala +++ /dev/null @@ -1,15 +0,0 @@ -package cwl.ontology - -import com.typesafe.config.Config -import common.validation.Validation._ -import net.ceedubs.ficus.Ficus._ - -case class CacheConfiguration(maxSize: Long) - -object CacheConfiguration { - def apply(config: Config): CacheConfiguration = { - validate(config.getAs[Long]("max-size").getOrElse(0L)) - .map(new CacheConfiguration(_)) - .unsafe("Ontology cache configuration") - } -} diff --git a/cwl/src/main/scala/cwl/ontology/OntologyConfiguration.scala b/cwl/src/main/scala/cwl/ontology/OntologyConfiguration.scala deleted file mode 100644 index 5382ac35b91..00000000000 --- a/cwl/src/main/scala/cwl/ontology/OntologyConfiguration.scala +++ /dev/null @@ -1,27 +0,0 @@ -package cwl.ontology - -import cats.syntax.apply._ -import com.typesafe.config.Config -import common.util.Backoff -import common.validation.ErrorOr.ErrorOr -import common.validation.Validation._ -import net.ceedubs.ficus.Ficus._ - -import scala.concurrent.duration.FiniteDuration - -case class OntologyConfiguration(retries: Option[Int], backoff: Backoff, poolSize: Int) - -object OntologyConfiguration { - def apply(config: Config): OntologyConfiguration = { - val retries = validate { Option(config.as[Int]("retries")) } - val backoff = validate { - Backoff.staticBackoff( - config.as[FiniteDuration]("backoff-time") - ) - } - val poolSize = validate { config.as[Int]("pool-size") } - - val validated: ErrorOr[OntologyConfiguration] = (retries, backoff, poolSize).mapN(OntologyConfiguration.apply) - validated.unsafe("Ontology configuration") - } -} diff --git a/cwl/src/main/scala/cwl/ontology/Schema.scala b/cwl/src/main/scala/cwl/ontology/Schema.scala deleted file mode 100644 index a2ed13d7bf3..00000000000 --- a/cwl/src/main/scala/cwl/ontology/Schema.scala +++ /dev/null @@ -1,180 +0,0 @@ -package cwl.ontology - -import java.util.concurrent.Executors - -import cats.effect.IO -import cats.syntax.traverse._ -import cats.instances.list._ -import com.google.common.cache.{Cache, CacheBuilder} -import com.typesafe.config.{Config, ConfigFactory} -import com.typesafe.scalalogging.Logger -import common.util.IORetry -import common.util.IORetry.StatefulIoError -import common.validation.ErrorOr._ -import common.validation.Validation._ -import cwl.ontology.Schema._ -import mouse.all._ -import net.ceedubs.ficus.Ficus._ -import org.semanticweb.owlapi.apibinding.OWLManager -import org.semanticweb.owlapi.model._ -import org.semanticweb.owlapi.model.parameters.OntologyCopy -import org.semanticweb.owlapi.reasoner.structural.StructuralReasonerFactory -import org.semanticweb.owlapi.reasoner.{OWLReasoner, OWLReasonerFactory} -import org.semanticweb.owlapi.util.OWLAPIStreamUtils -import org.slf4j.LoggerFactory - -import scala.jdk.CollectionConverters._ -import scala.concurrent.ExecutionContext -import scala.util.Try - -/** - * OWL/RDF Schema lookup. - * - * @param schemaIris IRI paths to OWL/RDF schemas. - * @param namespaces Additional/Override namespace prefixes. - */ -case class Schema(schemaIris: Seq[String], - namespaces: Map[String, String], - ontologyManager: OWLOntologyManager = OWLManager.createOWLOntologyManager, - reasonerFactory: OWLReasonerFactory = new StructuralReasonerFactory) { - - /** - * Returns a full IRI based on a full or abbreviated IRI. - * - * A full IRI wrapped in < and > will not be checked for abbreviations but will be returned without the wrapper. - * - * @see https://www.w3.org/TR/owl2-syntax/#IRIs - */ - def fullIri(name: String): String = getIri(name).getIRIString - - /** - * Returns true if child is an equivalent of ancestor, and if not then recursively checks if any of the child's - * super classes are an equivalent of ancestor. - */ - def isSubClass(child: String, ancestor: String): Boolean = { - val childIri: IRI = getIri(child) - val ancestorIri: IRI = getIri(ancestor) - val childClass: OWLClass = dataFactory.getOWLClass(childIri) - val ancestorClass: OWLClass = dataFactory.getOWLClass(ancestorIri) - val schemaReasoner: OWLReasoner = reasonerFactory.createReasoner(schemaOntology) - try { - Schema.isSubClass(schemaReasoner, childClass, ancestorClass) - } finally { - schemaReasoner.dispose() - } - } - - private val dataFactory: OWLDataFactory = ontologyManager.getOWLDataFactory - private val schemaOntology: OWLOntology = ontologyManager.createOntology() - private val schemaPrefixManager: PrefixManager = - ontologyManager.getOntologyFormat(schemaOntology).asPrefixOWLDocumentFormat - - { - def addToSchema(originalOntology: OWLOntology): Unit = { - schemaOntology.addAxioms(originalOntology.axioms()) - val originalOntologyFormat: OWLDocumentFormat = ontologyManager.getOntologyFormat(originalOntology) - if (originalOntologyFormat.isPrefixOWLDocumentFormat) - schemaPrefixManager.copyPrefixesFrom(originalOntologyFormat.asPrefixOWLDocumentFormat) - } - - val errorOr: ErrorOr[Unit] = for { - ontologies <- schemaIris.toList.traverse(loadOntologyFromIri(ontologyManager)) - _ = ontologies.foreach(addToSchema) - } yield () - errorOr.toTry("Error loading schemas").get - - // Add any namespace overrides - namespaces foreach { - case (prefixName, prefix) => schemaPrefixManager.setPrefix(prefixName, prefix) - } - } - - /** - * Returns the full IRI for the name using the prefixManager. - * - * TODO: Not 100% sure why you can't ask owl-api for an iri-with-prefix and have it looked up automatically. - * - * There does seem to be a difference between abbreviated and full IRIs in the official spec, where full IRIs are - * wrapped in < >. But this doesn't seem to be the format used by CWL nor owl-api. - * - * Please update this comment if/when one knows the correct behavior. - * - * @see https://www.w3.org/TR/owl2-syntax/#IRIs - */ - private def getIri(name: String): IRI = { - Try(schemaPrefixManager.getIRI(name)).getOrElse(IRI.create(name)) - } -} - -object Schema { - // Extending StrictLogging creates a circular dependency here for some reason, so making the logger ourselves - private val logger: Logger = Logger(LoggerFactory.getLogger(getClass.getName)) - private [ontology] val ontologyConfig = ConfigFactory.load.as[Config]("ontology") - private val ontologyConfiguration = OntologyConfiguration(ontologyConfig) - private [ontology] val cacheConfig = ontologyConfig.getAs[Config]("cache") - - // Simple cache to avoid reloading the same ontologies too often - private val ontologyCache = cacheConfig.map(makeOntologyCache) - - private implicit val statefulIoError = StatefulIoError.noop[Unit] - private implicit val timer = cats.effect.IO.timer(ExecutionContext.fromExecutor(Executors.newFixedThreadPool(ontologyConfiguration.poolSize))) - - private [ontology] def makeOntologyCache(config: Config): Cache[IRI, OWLOntology] = { - val cacheConfig = CacheConfiguration(config) - logger.info(s"Ontology cache size: ${cacheConfig.maxSize}") - CacheBuilder.newBuilder() - .maximumSize(cacheConfig.maxSize) - .build[IRI, OWLOntology]() - } - - /** - * Returns the absolute path for a file, possibly relative to parent. - */ - def getIriPath(parent: String, path: String): String = IRI.create(parent).resolve(path).getIRIString - - /** - * Load an ontology either from an IRI. - */ - private [ontology] def loadOntologyFromIri(ontologyManager: OWLOntologyManager, cache: Option[Cache[IRI, OWLOntology]] = ontologyCache)(schemaIri: String): ErrorOr[OWLOntology] = { - validate { - val iri = IRI.create(schemaIri) - cache.flatMap(_.getIfPresent(iri) |> Option.apply) match { - case Some(ontology) => - ontologyManager.copyOntology(ontology, OntologyCopy.DEEP) - case _ => - logger.info(s"Loading ${iri.toURI.toString}") - val ontology = loadOntologyFromIri(ontologyManager, iri) - cache.foreach(_.put(iri, ontology)) - ontology - } - } - } - - // Loading the ontology can fail transiently, so put retires around it. See https://github.com/protegeproject/webprotege/issues/298 - private [ontology] def loadOntologyFromIri(ontologyManager: OWLOntologyManager, iri: IRI): OWLOntology = { - val load = IO { ontologyManager.loadOntologyFromOntologyDocument(iri) } - IORetry.withRetry[OWLOntology, Unit](load, (), ontologyConfiguration.retries, ontologyConfiguration.backoff).unsafeRunSync() - } - - /** - * Returns true if child is an equivalent of ancestor, and if not then recursively checks if any of the child's - * super classes are an equivalent of ancestor. - */ - private def isSubClass(reasoner: OWLReasoner, childClass: OWLClass, ancestorClass: OWLClass): Boolean = { - val equivalent: Set[OWLClass] = reasoner.getEquivalentClasses(childClass).asScala.toSet + childClass - if (equivalent.contains(ancestorClass)) { - true - } else { - val parentClasses: Set[OWLClass] = for { - equivalentClass <- equivalent - parentClass <- OWLAPIStreamUtils - .asSet(reasoner.getSuperClasses(equivalentClass).entities) - .asScala - .toSet[OWLClass] - } yield parentClass - parentClasses.collect({ - case superClass: OWLClass if isSubClass(reasoner, superClass, ancestorClass) => superClass - }).nonEmpty - } - } -} diff --git a/cwl/src/main/scala/cwl/package.scala b/cwl/src/main/scala/cwl/package.scala deleted file mode 100644 index 96796712f1e..00000000000 --- a/cwl/src/main/scala/cwl/package.scala +++ /dev/null @@ -1,154 +0,0 @@ - -import cats.data.ReaderT -import common.Checked -import common.validation.Checked._ -import common.validation.ErrorOr._ -import cwl.CwlType._ -import cwl.ExpressionEvaluator.{ECMAScriptExpression, ECMAScriptFunction} -import cwl.command.ParentName -import cwl.ontology.Schema -import shapeless._ -import wom.executable.Executable -import wom.expression.IoFunctionSet -import wom.types._ -import wom.values.WomEvaluatedCallInputs - -import scala.util.{Failure, Success, Try} - -/** - * This package is intended to parse all CWL files. - * - * It makes heavy use of Circe YAML/Json auto derivation feature and - * Circe modules that support the Scala libraries shapeless and Refined. - * - * The [[https://oss.sonatype.org/service/local/repositories/releases/archive/com/chuusai/shapeless_2.12/2.3.2/shapeless_2.12-2.3.2-javadoc.jar/!/shapeless/Coproduct.html shapeless.coproduct]] feature allows us to specify a large - * number of potential types to be parsed. A.k.a. a "disjunction" or "OR" relationship amongst these types. - * - * The [[https://github.com/fthomas/refined/blob/master/modules/core/shared/src/main/scala/eu/timepit/refined/string.scala MatchesRegex]] "refined type" is used - * to enforce structure upon String values in the CWL Yaml. Shapeless' Witness type - * is used to declare a type containing a String literal. - * - * @see CWL Specification - * @see circe - * @see circe-yaml - * @see Refined - * @see Shapeless - */ -package object cwl extends TypeAliases { - - type CwlFile = Array[Cwl] :+: Cwl :+: CNil - type Cwl = Workflow :+: CommandLineTool :+: ExpressionTool :+: CNil - - object Cwl { - object Workflow { def unapply(cwl: Cwl): Option[Workflow] = cwl.select[Workflow] } - object CommandLineTool { def unapply(cwl: Cwl): Option[CommandLineTool] = cwl.select[CommandLineTool] } - object ExpressionTool { def unapply(cwl: Cwl): Option[ExpressionTool] = cwl.select[ExpressionTool] } - } - - def cwlTypeToWomType : CwlType => WomType = { - case CwlType.Any => WomAnyType - case Null => WomNothingType - case Boolean => WomBooleanType - case Int => WomIntegerType - case Long => WomLongType - case Float => WomFloatType - case Double => WomFloatType - case String => WomStringType - case CwlType.File => WomMaybePopulatedFileType - case CwlType.Directory => WomMaybeListedDirectoryType - } - - object StringOrExpression { - object String { - def unapply(soe: StringOrExpression): Option[String] = soe.select[String] - } - object Expression { - def unapply(soe: StringOrExpression): Option[Expression] = soe.select[Expression] - } - object ECMAScriptExpression { - def unapply(soe: StringOrExpression): Option[ECMAScriptExpression] = soe.select[Expression].flatMap(_.select[ECMAScriptExpression]) - } - object ECMAScriptFunction { - def unapply(soe: StringOrExpression): Option[ECMAScriptFunction] = soe.select[Expression].flatMap(_.select[ECMAScriptFunction]) - } - } - - object Expression { - object ECMAScriptExpression { - def unapply(soe: Expression): Option[ECMAScriptExpression] = soe.select[ECMAScriptExpression] - } - object ECMAScriptFunction { - def unapply(soe: Expression): Option[ECMAScriptFunction] = soe.select[ECMAScriptFunction] - } - } - - type WomTypeMap = Map[String, WomType] - - type RequirementsValidator = Requirement => ErrorOr[Requirement] - import cats.syntax.validated._ - val AcceptAllRequirements: RequirementsValidator = _.validNel - - implicit class CwlHelper(val cwl: Cwl) extends AnyVal { - def womExecutable(validator: RequirementsValidator, inputsFile: Option[String], ioFunctions: IoFunctionSet, strictValidation: Boolean): Checked[Executable] = { - def executable = cwl match { - case Cwl.Workflow(w) => w.womExecutable(validator, inputsFile, ioFunctions, strictValidation) - case Cwl.CommandLineTool(clt) => clt.womExecutable(validator, inputsFile, ioFunctions, strictValidation) - case Cwl.ExpressionTool(et) => et.womExecutable(validator, inputsFile, ioFunctions, strictValidation) - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - Try(executable) match { - case Success(s) => s - case Failure(f) => f.getMessage.invalidNelCheck - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - } - - def requiredInputs: Map[String, WomType] = { - implicit val parent = ParentName.empty - - cwl match { - case Cwl.Workflow(w) => selectWomTypeInputs(w.inputs collect { - case i if i.`type`.isDefined => FullyQualifiedName(i.id).id -> i.`type`.get - }) - case Cwl.CommandLineTool(clt) => selectWomTypeInputs(clt.inputs collect { - case i if i.`type`.isDefined => FullyQualifiedName(i.id).id -> i.`type`.get - }) - case Cwl.ExpressionTool(et) => selectWomTypeInputs(et.inputs collect { - case i if i.`type`.isDefined => FullyQualifiedName(i.id).id -> i.`type`.get - }) - case oh => throw new Exception(s"Programmer Error! Unexpected case match: $oh") - } - } - - def schemaOption: Option[Schema] = cwl.fold(CwlSchemaOptionPoly) - - private def selectWomTypeInputs(myriadInputMap: Array[(String, MyriadInputType)]): Map[String, WomType] = { - (myriadInputMap collect { - case (key, MyriadInputType.WomType(w)) => key -> w - }).toMap - } - } - - object CwlSchemaOptionPoly extends Poly1 { - implicit val caseWorkflow: Case.Aux[Workflow, Option[Schema]] = at { - workflow => getSchema(workflow.`$schemas`, workflow.`$namespaces`) - } - implicit val caseCommandLineTool: Case.Aux[CommandLineTool, Option[Schema]] = at { - commandLineTool => getSchema(commandLineTool.`$schemas`, commandLineTool.`$namespaces`) - } - implicit val caseExpressionTool: Case.Aux[ExpressionTool, Option[Schema]] = at { - expressionTool => getSchema(expressionTool.`$schemas`, expressionTool.`$namespaces`) - } - - private def getSchema(schemasOption: Option[Array[String]], - namespacesOption: Option[Map[String, String]]): Option[Schema] = { - schemasOption.map(a => Schema(a.toIndexedSeq, namespacesOption getOrElse Map.empty)) - } - } - - type ExpressionLib = Vector[String] - - type Inputs = (RequirementsAndHints, ExpressionLib, WomEvaluatedCallInputs) - - type CommandPartExpression[A] = ReaderT[ErrorOr, Inputs, A] -} diff --git a/cwl/src/main/scala/cwl/preprocessor/CwlCanonicalizer.scala b/cwl/src/main/scala/cwl/preprocessor/CwlCanonicalizer.scala deleted file mode 100644 index b369d404c18..00000000000 --- a/cwl/src/main/scala/cwl/preprocessor/CwlCanonicalizer.scala +++ /dev/null @@ -1,249 +0,0 @@ -package cwl.preprocessor - -import cats.effect.{ContextShift, IO} -import cats.syntax.parallel._ -import cats.instances.list._ -import common.validation.ErrorOr.ErrorOr -import common.validation.IOChecked._ -import common.validation.Validation._ -import cwl.preprocessor.CwlReference.EnhancedCwlId -import cwl.preprocessor.CwlPreProcessor._ -import io.circe.Json -import io.circe.optics.JsonPath._ -import cwl.preprocessor.CwlCanonicalizer._ - -/** - * The real guts of the CWL pre-processor is taking a CWL reference and producing a single, self-contained JSON from it. - */ -private [preprocessor] class CwlCanonicalizer(saladFunction: SaladFunction)(implicit cs: ContextShift[IO]) { - - def getCanonicalCwl(reference: CwlReference, - namespacesJsonOption: Option[Json] = None, - schemasJsonOption: Option[Json] = None): IOChecked[Json] = { - flattenCwlReferenceInner( - reference, - Map.empty, - Map.empty, - Set.empty, - namespacesJsonOption, - schemasJsonOption).map(_.processedJson) - } - - /** - * Flatten the cwl reference given already known processed references. - */ - private def flattenCwlReferenceInner(cwlReference: CwlReference, - unProcessedReferences: UnProcessedReferences, - processedReferences: ProcessedReferences, - breadCrumbs: Set[CwlReference], - namespacesJsonOption: Option[Json], - schemasJsonOption: Option[Json]): IOChecked[ProcessedJsonAndDependencies] = { - /* - * Salad and parse from a CWL reference into a Json object - */ - def saladAndParse(ref: CwlReference): IOChecked[Json] = for { - saladed <- saladFunction(ref) - saladedJson <- parseJson(saladed) - } yield saladedJson - - for { - // parse the file containing the reference - parsed <- saladAndParse(cwlReference) - // Get a Map[CwlReference, Json] from the parsed file. If the file is a JSON object and only contains one node, the map will only have 1 element - newUnProcessedReferences = mapIdToContent(parsed).toMap - // The reference json in the file - referenceJson <- newUnProcessedReferences - .collectFirst({ case (ref, json) if ref.pointerWithinFile == cwlReference.pointerWithinFile => json }) - .toIOChecked(s"Cannot find a tool or workflow with ID '${cwlReference.pointerWithinFile}' in file ${cwlReference.pathAsString}'s set: [${newUnProcessedReferences.keySet.mkString(", ")}]") - // Process the reference json - processed <- flattenJson( - referenceJson, - newUnProcessedReferences ++ unProcessedReferences, - processedReferences, - breadCrumbs + cwlReference, - namespacesJsonOption, - schemasJsonOption - ) - } yield processed - } - - /** - * Given a Json representing a tool or workflow, flattens it and return the other processed references that were generated. - * - * NB: Flatten here means two things: - * - Find references within the CWL and convert them into 'local' links - * - Create a map of canonical links to JSON CWL content - * - * @param saladedJson json to process - * @param unProcessedReferences references that have been parsed and saladed (we have the json), but not flattened yet - * @param processedReferences references that are fully flattened - * @param breadCrumbs list of references that brought us here - * @param namespacesJsonOption Namespaces from the original json - * @param schemasJsonOption Schemas from the original json - */ - private def flattenJson(saladedJson: Json, - unProcessedReferences: UnProcessedReferences, - processedReferences: ProcessedReferences, - breadCrumbs: Set[CwlReference], - namespacesJsonOption: Option[Json], - schemasJsonOption: Option[Json]): IOChecked[ProcessedJsonAndDependencies] = { - /* - * Given a reference from a step's run field, flattens it and return it - * @param unProcessedReferences references that have been parsed and saladed (we have the json), but not flattened yet. - * @param checkedProcessedReferences references that are fully processed - * @param cwlReference reference being processed - * @return a new ProcessedReferences Map including this cwlReference processed along with all the dependencies - * that might have been processed recursively. - */ - def processCwlRunReference(checkedProcessedReferences: IOChecked[ProcessedReferences], - cwlReference: CwlReference): IOChecked[ProcessedReferences] = { - def processReference(processedReferences: ProcessedReferences) = { - - val result: IOChecked[ProcessedJsonAndDependencies] = unProcessedReferences.get(cwlReference) match { - case Some(unProcessedReferenceJson) => - // Found the json in the unprocessed map, no need to reparse the file, just flatten this json - flattenJson( - unProcessedReferenceJson, - unProcessedReferences, - processedReferences, - breadCrumbs, - namespacesJsonOption, - schemasJsonOption - ) - case None => - // This is the first time we're seeing this reference, we need to parse its file and flatten it - flattenCwlReferenceInner( - cwlReference, - unProcessedReferences, - processedReferences, - breadCrumbs, - namespacesJsonOption, - schemasJsonOption - ) - } - - result map { - // Return everything we've got (the previously known "processedReferences" + our new processed reference + everything that was processed to get to it) - case ProcessedJsonAndDependencies(processed, newReferences) => processedReferences ++ newReferences + (cwlReference -> processed) - } - } - - def processIfNeeded(processedReferences: ProcessedReferences): IOChecked[ProcessedReferences] = { - // If the reference has already been processed, no need to do anything - if (processedReferences.contains(cwlReference)) processedReferences.validIOChecked - // If the reference is in the bread crumbs it means we circled back to it: fail the pre-processing - else if (breadCrumbs.contains(cwlReference)) s"Found a circular dependency on $cwlReference".invalidIOChecked - // Otherwise let's see if we already have the json for it or if we need to process the file - else processReference(processedReferences) - } - - for { - processedReferences <- checkedProcessedReferences - newReferences <- processIfNeeded(processedReferences) - } yield newReferences - } - - def addJsonKeyValue(originalJson: Json, key: String, valueOption: Option[Json]): Json = { - valueOption match { - case Some(value) => originalJson.mapObject(_.add(key, value)) - case None => originalJson - } - } - - val namespacesJson = addJsonKeyValue(saladedJson, JsonKeyNamespaces, namespacesJsonOption) - val schemasJson = addJsonKeyValue(namespacesJson, JsonKeySchemas, schemasJsonOption) - - import cats.syntax.apply._ - - // Take the processed runs and inject them in the json - def inlineProcessedJsons(newKnownReferences: ProcessedReferences, inlinedRunWorkflows: Map[String, ProcessedJsonAndDependencies]) = { - // Provide a function to swap the run reference with its json content - val lookupFunction: Json => Json = { - json: Json => { - val fromRunReferenceMap = for { - asString <- json.asString - reference <- asString.asReference - embeddedJson <- newKnownReferences.get(reference) - } yield embeddedJson - - val fromInlinedWorkflow = for { - asObject <- json.asObject - id <- asObject.kleisli("id") - idAsString <- id.asString - embeddedJson <- inlinedRunWorkflows.get(idAsString) - } yield embeddedJson.processedJson - - fromRunReferenceMap.orElse(fromInlinedWorkflow).getOrElse(json) - } - } - - val flattenedJson = root.steps.each.run.json.modify(lookupFunction)(schemasJson) - - ProcessedJsonAndDependencies(flattenedJson, newKnownReferences ++ inlinedRunWorkflows.values.flatMap(_.processedDependencies)) - } - - /* - * Given a json, collects all "steps.run" values that are JSON Strings, and convert them to CwlReferences. - * A saladed JSON is assumed. - */ - def findRunReferences(json: Json): List[CwlReference] = { - json.asArray match { - case Some(cwls) => cwls.toList.flatMap(findRunReferences) - case _ => root.steps.each.run.string.getAll(json).flatMap(_.asReference).distinct - } - } - - /* - * Given a json, collects all "steps.run" values that are JSON Objects representing a workflow. - * A saladed JSON is assumed. - * @return a Map[String, Json], where the key is the cwl id of the workflow, and the value its content - */ - def findRunInlinedWorkflows(json: Json): ErrorOr[Map[String, Json]] = { - import cats.syntax.traverse._ - - json.asArray match { - case Some(cwls) => cwls.toList - .flatTraverse(findRunInlinedWorkflows(_).map(_.toList)) - .map(_.toMap) - case _ => - // Look for all the "run" steps that are json objects - root.steps.each.run.obj.getAll(json) - .map(Json.fromJsonObject) - // Only keep the workflows (CommandLineTools don't have steps so no need to process them) - .filter(root.`class`.string.exist(_.equalsIgnoreCase("Workflow"))) - .traverse[ErrorOr, (String, Json)]( obj => - // Find the id of the workflow - root.id.string.getOption(obj) - .toErrorOr("Programmer error: Workflow did not contain an id. Make sure the cwl has been saladed") - .map(_ -> obj) - ).map(_.toMap) - } - } - - // Recursively process the run references (where run is a string pointing to another Workflow / Tool) - // TODO: it would be nice to accumulate failures here somehow (while still folding and be able to re-use - // successfully processed references, so I don't know if ErrorOr would work) - val processedRunReferences: IOChecked[ProcessedReferences] = findRunReferences(schemasJson).foldLeft(processedReferences.validIOChecked)(processCwlRunReference) - - // Recursively process the inlined run workflows (where run is a Json object representing a workflow) - val processedInlineReferences: IOChecked[Map[String, ProcessedJsonAndDependencies]] = (for { - inlineWorkflowReferences <- findRunInlinedWorkflows(saladedJson).toIOChecked - flattenedWorkflows <- inlineWorkflowReferences.toList.parTraverse[IOChecked, (String, ProcessedJsonAndDependencies)]({ - case (id, value) => flattenJson(value, unProcessedReferences, processedReferences, breadCrumbs, namespacesJsonOption, schemasJsonOption).map(id -> _) - }) - } yield flattenedWorkflows).map(_.toMap) - - // Replace the unprocessed runs with their processed value - (processedRunReferences, processedInlineReferences).tupled.map(Function.tupled(inlineProcessedJsons)) - } -} - -private object CwlCanonicalizer { - /** - * A Cwl json that has been processed (saladed and flattened), as well as its processed dependencies. - */ - final case class ProcessedJsonAndDependencies(processedJson: Json, processedDependencies: ProcessedReferences) - - final type UnProcessedReferences = Map[CwlReference, Json] - final type ProcessedReferences = Map[CwlReference, Json] -} diff --git a/cwl/src/main/scala/cwl/preprocessor/CwlPreProcessor.scala b/cwl/src/main/scala/cwl/preprocessor/CwlPreProcessor.scala deleted file mode 100644 index 9855ee73953..00000000000 --- a/cwl/src/main/scala/cwl/preprocessor/CwlPreProcessor.scala +++ /dev/null @@ -1,236 +0,0 @@ -package cwl.preprocessor - -import java.util.concurrent.Executors - -import cats.data.NonEmptyList -import cats.effect.{ContextShift, IO} -import cats.syntax.either._ -import common.validation.IOChecked._ -import cwl.CwlDecoder -import cwl.ontology.Schema -import cwl.preprocessor.CwlPreProcessor._ -import cwl.preprocessor.CwlReference.EnhancedCwlId -import io.circe.optics.JsonPath._ -import io.circe.{Json, JsonNumber, JsonObject} -import mouse.all._ -import org.slf4j.LoggerFactory -import wom.util.YamlUtils - -import scala.concurrent.ExecutionContext - -/** - * Class to create a standalone version of a CWL file. - * - * NB: Want to use the pre-processor? Use preProcessCwl(ref: CwlReference) - * - * @param saladFunction function that takes a file and produce a saladed version of the content - */ -class CwlPreProcessor(saladFunction: SaladFunction = saladCwlFile) { - - private val ec: ExecutionContext = ExecutionContext.fromExecutor(Executors.newFixedThreadPool(5)) - private implicit val cs = IO.contextShift(ec) - - /** - * This is THE main entry point into the CWL pre-processor. Takes a CWL reference and - * returns a canonical JSON version with all references resolved. - * - * @param ref The reference to the CWL to pre-process - * @return A canonical JSON representation of the CWL with all internal references expanded in-place - */ - def preProcessCwl(ref: CwlReference): IOChecked[Json] = ref match { - case file: CwlFileReference => preProcessCwlFile(file) - case other => preProcessRemoteCwl(other) - } - - /** - * Convenience method to get the processed workflow as a string. - */ - def preProcessCwlToString(cwlReference: CwlReference): IOChecked[String] = preProcessCwl(cwlReference).map(_.printCompact) - - def preProcessInputFiles(inputContent: String, mappingFunction: String => String): IOChecked[String] = for { - parsed <- parseYaml(inputContent) - mapped = parsed |> mapFilesAndDirectories(mappingFunction) |> mapNumbers - } yield mapped.printCompact - - /** - * Pre-process a CWL file and create a standalone, runnable (given proper inputs), inlined version of its content. - * - * The general idea is to work on CwlReferences, starting from the one coming to this function in the form of file and optional root. - * The goal is to look at the steps in this workflow that point to other references, and recursively flatten them until we can replace the step with - * its flat version. - * - * There are 3 pieces of information that are carried around during this process: - * 1) ProcessedReferences: A Map[CwlReference, Json] of CwlReference for which we have the fully processed (saladed AND flattened) Json value. - * - * 2) UnProcessedReferences: A Map[CwlReference, Json] of CwlReference for which we have the saladed but NOT flattened Json value. - * This can happen because a file can contain multiple tools / workflows. When we salad / parse this file, we get (CwlReference, Json) pairs - * for all the workflow / tools in the file, but they are not flattened yet. - * We keep this to avoid having to re-salad / re-parse files unnecessarily. - * - * 3) BreadCrumb: A Set[CwlReference] used to follow the trail of CwlReferences that we are processing as we recurse down. - * This is used to be able to detect circular dependencies (if the cwlReference being processed is in that set, then we have a circular dependency) . - * - */ - private def preProcessCwlFile(reference: CwlFileReference): IOChecked[Json] = { - - def absoluteSchemaPaths(json: Json): Json = { - json mapArray { - _ map absoluteSchemaPaths - } mapString { - Schema.getIriPath(reference.fullReference, _) - } - } - - // NB the JSON here is only used to decide whether or not to flatten. If we do decide to flatten we throw away the - // json and request a canonical version from the CwlCanonicalizer. - def flattenOrByPass(json: Json): IOChecked[Json] = { - def flatten(json: Json): IOChecked[Json] = { - val cwlReferenceFlattener = new CwlCanonicalizer(saladFunction) - val namespacesJsonOption: Option[Json] = json.asObject.flatMap(_.kleisli(JsonKeyNamespaces)) - val schemasJsonOption: Option[Json] = json.asObject.flatMap(_.kleisli(JsonKeySchemas)).map(absoluteSchemaPaths) - - cwlReferenceFlattener.getCanonicalCwl( - reference, - namespacesJsonOption, - schemasJsonOption - ) - } - - def bypass(alreadyCanonicalJson: Json): IOChecked[Json] = alreadyCanonicalJson.validIOChecked - - val fileContentReference = for { - asObject <- json.asObject - fileContentId <- asObject.kleisli("id") - stringId <- fileContentId.asString - fileContentReference <- CwlReference.fromString(stringId) - } yield fileContentReference - - fileContentReference match { - // This by passes the pre-processing if the file already has an id for which the file part doesn't match the path of the file - // passed to this function, as this would indicate that it has already been saladed and pre-processed. - case Some(CwlFileReference(file, _)) if !file.equals(reference.file) => bypass(json) - case _ => flatten(json) - } - } - - for { - original <- parseYaml(reference.file.contentAsString) - flattened <- flattenOrByPass(original) - } yield flattened - } - - // Like 'File', except that we don't read any contents before passing the path over to cwltool to canonicalize. - private def preProcessRemoteCwl(reference: CwlReference)(implicit cs: ContextShift[IO]): IOChecked[Json] = { - val cwlCanonicalizer = new CwlCanonicalizer(saladFunction) - cwlCanonicalizer.getCanonicalCwl(reference) - } -} - -object CwlPreProcessor { - private val Log = LoggerFactory.getLogger("CwlPreProcessor") - - private [preprocessor] type SaladFunction = CwlReference => IOChecked[String] - private [preprocessor] val JsonKeyNamespaces = s"$$namespaces" - private [preprocessor] val JsonKeySchemas = s"$$schemas" - - private def saladSpinner(doLogging: Boolean): SaladFunction = ref => { - if (doLogging) { - Log.info(s"Pre-Processing ${ref.pathAsString}") - } - - CwlDecoder.saladCwlFile(ref) - } - - private [preprocessor] val saladCwlFile: SaladFunction = saladSpinner(true) - private val saladCwlFileWithoutLogging: SaladFunction = saladSpinner(false) - - implicit class PrintableJson(val json: Json) extends AnyVal { - def printCompact = io.circe.Printer.noSpaces.print(json) - } - - def noLogging = new CwlPreProcessor(saladCwlFileWithoutLogging) - - // Fold over a json recursively and prefix all files - def mapFilesAndDirectories(mappingFunction: String => String)(json: Json): Json = { - // Function to check if the given json has the provided key / value pair - def hasKeyValue(key: String, value: String): Json => Boolean = { - root.selectDynamic(key).string.exist(_.equalsIgnoreCase(value)) - } - - // Return true if the given json object represents a File - def isFile(obj: JsonObject) = hasKeyValue("class", "File")(Json.fromJsonObject(obj)) - - // Return true if the given json object represents a Directory - def isDirectory(obj: JsonObject) = hasKeyValue("class", "Directory")(Json.fromJsonObject(obj)) - - // Modify the string at "key" using the mappingFunction - def mapStringValue(key: String, mappingFunction: String => String): Json => Json = root.selectDynamic(key).string.modify(mappingFunction) - - // Map "location" and "default" - def prefix(mappingFunction: String => String): Json => Json = mapStringValue("location", mappingFunction).compose(mapStringValue("path", mappingFunction)) - - // Prefix the location or path in the json object if it's a file or directory, otherwise recurse over its fields - def prefixObject(mappingFunction: String => String)(obj: JsonObject): Json = { - // If the object is file or a directory, prefix it with the gcs prefix - if (isFile(obj) || isDirectory(obj)) { - prefix(mappingFunction)(Json.fromJsonObject(obj)) - // Even if it's a file it may have secondary files. So keep recursing on its fields - .mapObject(_.mapValues(mapFilesAndDirectories(mappingFunction))) - } - // Otherwise recursively process its fields - else Json.fromJsonObject(obj.mapValues(mapFilesAndDirectories(mappingFunction))) - } - - json.fold( - jsonNull = json, - jsonBoolean = _ => json, - jsonNumber = _ => json, - jsonString = _ => json, - jsonObject = prefixObject(mappingFunction), - jsonArray = arr => Json.arr(arr.map(mapFilesAndDirectories(mappingFunction)): _*) - ) - } - - private [preprocessor] def mapNumbers(json: Json): Json = { - // Circumvent Circe's scientific format for numbers: convert to a JSON String without exponential notation. - def nonScientificNumberFormatting(jsonNumber: JsonNumber): Json = { - val conversions = LazyList[JsonNumber => Option[Any]]( - _.toBigInt.map(_.longValue), - _.toBigDecimal.map(_.doubleValue), - Function.const(Option("null"))) - - // The `get` is safe because `Option("null")` guarantees a match even if the other two Stream elements - // do not satisfy the predicate. - conversions.map(_.apply(jsonNumber)).find(_.isDefined).flatten.get.toString |> Json.fromString - } - - json.fold( - jsonNull = json, - jsonBoolean = _ => json, - jsonNumber = nonScientificNumberFormatting, - jsonString = _ => json, - jsonObject = _.mapValues(mapNumbers) |> Json.fromJsonObject, - jsonArray = _.map(mapNumbers) |> Json.fromValues - ) - } - - private [preprocessor] def parseJson(in: String): IOChecked[Json] = { - io.circe.parser.parse(in).leftMap(error => NonEmptyList.one(error.message)).toIOChecked - } - - private [preprocessor] def parseYaml(in: String): IOChecked[Json] = { - val yaml = YamlUtils.parse(in) - yaml.leftMap(error => NonEmptyList.one(error.message)).toIOChecked - } - - /** - * Given a json, collect all tools or workflows and map them with their reference id. - * A saladed JSON is assumed. - */ - private [preprocessor] def mapIdToContent(json: Json): List[(CwlReference, Json)] = { - json.asArray match { - case Some(cwls) => cwls.toList.flatMap(mapIdToContent) - case None => root.id.string.getOption(json).flatMap(_.asReference).map(_ -> json).toList - } - } -} diff --git a/cwl/src/main/scala/cwl/preprocessor/CwlReference.scala b/cwl/src/main/scala/cwl/preprocessor/CwlReference.scala deleted file mode 100644 index badddf028b4..00000000000 --- a/cwl/src/main/scala/cwl/preprocessor/CwlReference.scala +++ /dev/null @@ -1,82 +0,0 @@ -package cwl.preprocessor - -import better.files.{File => BFile} -import cwl.preprocessor.CwlReference._ -import cwl.{FileAndId, FullyQualifiedName} -import cwl.command.ParentName - -sealed trait CwlReference { - def pathAsString: String - def pointerWithinFile: Option[String] - def changePointer(to: Option[String]): CwlReference - - private def pointerWithHash: String = pointerWithinFile.map(p => s"#$p").getOrElse("") - lazy val fullReference: String = s"$pathAsString$pointerWithHash" - - override def toString: String = fullReference -} - -/** - * Saladed CWLs reference other local CWL "node" (workflow or tool) using a URI as follow: - * file:///path/to/file/containing/node.cwl[#pointer_to_node] - * #pointer_to_node to node is optional, and will specify which workflow or tool is being targeted in the file. - * - * e.g: - * { - * "class": "Workflow", - * "id": "file:///path/to/workflow/workflow.cwl", - * ... - * "steps": [ - * { - * "run": "file:///path/to/workflow/multi_tools.cwl#my_tool", - * ... - * } - * ] - * } - * - * This snippet contains 2 references, one that is the ID of this workflow, the other one is the run step pointing to "my_tool" in "/path/to/workflow/multi_tools.cwl" - * - */ -final case class CwlFileReference(file: BFile, pointerWithinFile: Option[String]) extends CwlReference { - override val pathAsString: String = s"$LocalScheme${file.toString}" - override def changePointer(to: Option[String]): CwlReference = this.copy(pointerWithinFile = to) -} - -final case class CwlHttpReference(pathAsString: String, pointerWithinFile: Option[String]) extends CwlReference { - override def changePointer(to: Option[String]): CwlReference = this.copy(pointerWithinFile = to) -} - -object CwlReference { - val LocalScheme = "file://" - val HttpScheme = "http://" - val HttpsScheme = "https://" - - implicit class EnhancedCwlId(val id: String) extends AnyVal { - def asReference: Option[CwlReference] = CwlReference.fromString(id) - def stripFilePrefix = id.stripPrefix(LocalScheme) - } - - val ReferenceRegex = "(.*://)?([^#]*)(#(.*))?".r - - def fromString(in: String): Option[CwlReference] = { - in match { - case ReferenceRegex(scheme, path, _, pointerWithinFile) => - if (scheme == LocalScheme) { - FullyQualifiedName.maybeApply(in)(ParentName.empty) match { - case Some(FileAndId(file, _, _)) => Option(CwlFileReference(BFile(file.stripFilePrefix), Option(pointerWithinFile))) - case _ => Option(CwlFileReference(BFile(in.stripFilePrefix), Option(pointerWithinFile))) - } - } else if (scheme == HttpScheme || scheme == HttpsScheme) { - Option(CwlHttpReference(s"$scheme$path", Option(pointerWithinFile))) - } else { - None - } - } - } -} - -object CwlFileReference { - def apply(file: BFile, pointer: Option[String]) = { - new CwlFileReference(file, pointer) - } -} diff --git a/cwl/src/main/scala/cwl/requirement/RequirementToAttributeMap.scala b/cwl/src/main/scala/cwl/requirement/RequirementToAttributeMap.scala deleted file mode 100644 index 1a727837172..00000000000 --- a/cwl/src/main/scala/cwl/requirement/RequirementToAttributeMap.scala +++ /dev/null @@ -1,79 +0,0 @@ -package cwl.requirement - -import cwl._ -import shapeless.Poly1 -import wom.RuntimeAttributesKeys._ -import wom.expression.{ValueAsAnExpression, WomExpression} -import wom.values.{WomLong, WomString} - -object RequirementToAttributeMap extends Poly1 { - type ResourcesToExpressionMap = (Set[String], ExpressionLib) => Map[String, WomExpression] - implicit def fromJs: Case.Aux[InlineJavascriptRequirement, ResourcesToExpressionMap] = at[InlineJavascriptRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromSchemaDef: Case.Aux[SchemaDefRequirement, ResourcesToExpressionMap] = at[SchemaDefRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromDocker: Case.Aux[DockerRequirement, ResourcesToExpressionMap] = at[DockerRequirement] { - docker => (_,_) => docker.dockerPull.orElse(docker.dockerImageId).map({ pull => - DockerKey -> ValueAsAnExpression(WomString(pull)) - }).toMap - } - - implicit def fromSoftware: Case.Aux[SoftwareRequirement, ResourcesToExpressionMap] = at[SoftwareRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromInitialWorkDir: Case.Aux[InitialWorkDirRequirement, ResourcesToExpressionMap] = at[InitialWorkDirRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromEnvVar: Case.Aux[EnvVarRequirement, ResourcesToExpressionMap] = at[EnvVarRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromShellCommand: Case.Aux[ShellCommandRequirement, ResourcesToExpressionMap] = at[ShellCommandRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromResource: Case.Aux[ResourceRequirement, ResourcesToExpressionMap] = at[ResourceRequirement] { - resource => (inputNames, expressionLib) => - def toExpression(resourceRequirement: ResourceRequirementType) = - resourceRequirement.fold(ResourceRequirementToWomExpression).apply(inputNames, expressionLib) - - List( - // Map cpuMin to both cpuMin and cpu keys - resource.effectiveCoreMin.toList.map(toExpression).flatMap(min => List(CpuMinKey -> min, CpuKey -> min)), - resource.effectiveCoreMax.toList.map(toExpression).map(CpuMaxKey -> _), - // Map ramMin to both memoryMin and memory keys - resource.effectiveRamMin.toList.map(toExpression).flatMap(min => List(MemoryMinKey -> min, MemoryKey -> min)), - resource.effectiveRamMax.toList.map(toExpression).map(MemoryMaxKey -> _), - resource.effectiveTmpdirMin.toList.map(toExpression).map(TmpDirMinKey -> _), - resource.effectiveTmpdirMax.toList.map(toExpression).map(TmpDirMaxKey -> _), - resource.effectiveOutdirMin.toList.map(toExpression).map(OutDirMinKey -> _), - resource.effectiveOutdirMax.toList.map(toExpression).map(OutDirMaxKey -> _) - ).flatten.toMap - } - - implicit def fromInputResourceRequirement: Case.Aux[DnaNexusInputResourceRequirement, ResourcesToExpressionMap] = at[DnaNexusInputResourceRequirement] { - case DnaNexusInputResourceRequirement(_, indirMin) => (_, _) => indirMin.map(value => ValueAsAnExpression(WomLong(value))).map(DnaNexusInputDirMinKey -> _).toMap - } - - implicit def fromSubWorkflow: Case.Aux[SubworkflowFeatureRequirement, ResourcesToExpressionMap] = at[SubworkflowFeatureRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromScatter: Case.Aux[ScatterFeatureRequirement, ResourcesToExpressionMap] = at[ScatterFeatureRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromMultipleInput: Case.Aux[MultipleInputFeatureRequirement, ResourcesToExpressionMap] = at[MultipleInputFeatureRequirement] { - _ => (_,_) => Map.empty - } - - implicit def fromStepInput: Case.Aux[StepInputExpressionRequirement, ResourcesToExpressionMap] = at[StepInputExpressionRequirement] { - _ => (_,_) => Map.empty - } -} diff --git a/cwl/src/main/scala/cwl/requirement/ResourceRequirementToWomExpression.scala b/cwl/src/main/scala/cwl/requirement/ResourceRequirementToWomExpression.scala deleted file mode 100644 index 74727947657..00000000000 --- a/cwl/src/main/scala/cwl/requirement/ResourceRequirementToWomExpression.scala +++ /dev/null @@ -1,15 +0,0 @@ -package cwl.requirement - -import cwl.{Expression, ExpressionLib} -import shapeless.Poly1 -import wom.expression.{ValueAsAnExpression, WomExpression} -import wom.values.{WomLong, WomString} - -object ResourceRequirementToWomExpression extends Poly1 { - type ResourceRequirementStringSetToWomExpression = (Set[String], ExpressionLib) => WomExpression - implicit def fromLong: Case.Aux[Long, ResourceRequirementStringSetToWomExpression] = at[Long] { l => (_, _) => ValueAsAnExpression(WomLong(l)) } - implicit def fromString: Case.Aux[String, ResourceRequirementStringSetToWomExpression] = at[String] { s => (_, _) => ValueAsAnExpression(WomString(s)) } - implicit def fromExpression: Case.Aux[Expression, ResourceRequirementStringSetToWomExpression] = at[Expression] { e => (inputs, expressionLib) => - cwl.ECMAScriptWomExpression(e, inputs, expressionLib) - } -} diff --git a/cwl/src/test/resources/1st-tool.cwl b/cwl/src/test/resources/1st-tool.cwl deleted file mode 100755 index af0c4de297d..00000000000 --- a/cwl/src/test/resources/1st-tool.cwl +++ /dev/null @@ -1,9 +0,0 @@ -cwlVersion: v1.0 -class: CommandLineTool -baseCommand: echo -inputs: - message: - type: string - inputBinding: - position: 1 -outputs: [] diff --git a/cwl/src/test/resources/1st-workflow.cwl b/cwl/src/test/resources/1st-workflow.cwl deleted file mode 100644 index ce4b4b0bd71..00000000000 --- a/cwl/src/test/resources/1st-workflow.cwl +++ /dev/null @@ -1,23 +0,0 @@ -cwlVersion: v1.0 -class: Workflow -inputs: - inp: File - ex: string -outputs: - classout: - type: File - outputSource: compile/classfile - -steps: - untar: - run: tar-param.cwl - in: - tarfile: inp - extractfile: ex - out: [example_out] - - compile: - run: arguments.cwl - in: - src: untar/example_out - out: [classfile] diff --git a/cwl/src/test/resources/application.conf b/cwl/src/test/resources/application.conf deleted file mode 100644 index 8d12b670cd3..00000000000 --- a/cwl/src/test/resources/application.conf +++ /dev/null @@ -1,14 +0,0 @@ -akka { - log-dead-letters = "off" - loggers = ["akka.event.slf4j.Slf4jLogger"] -} - -ontology { - # Uncomment to enable caching of ontologies. Improves performance when loading ontologies from remote IRIs. - #cache { - # max-size = 20 - #} - retries = 3 - pool-size = 3 - backoff-time = 2 seconds -} diff --git a/cwl/src/test/resources/arguments.cwl b/cwl/src/test/resources/arguments.cwl deleted file mode 100644 index 730030e7437..00000000000 --- a/cwl/src/test/resources/arguments.cwl +++ /dev/null @@ -1,22 +0,0 @@ -cwlVersion: "v1.0" -class: "CommandLineTool" -label: "Example trivial wrapper for Java 7 compiler" -hints: - - dockerPull: "java:7-jdk" - class: "DockerRequirement" -baseCommand: "javac" -arguments: - - "-d" - - "$(runtime.outdir)" -inputs: - - type: "File" - inputBinding: - position: 1 - id: "file:///home/dan/wdl4s/arguments.cwl#src" -outputs: - - type: "File" - outputBinding: - glob: "*.class" - id: "file:///home/dan/wdl4s/arguments.cwl#classfile" -id: "file:///home/dan/wdl4s/arguments.cwl" -name: "file:///home/dan/wdl4s/arguments.cwl" diff --git a/cwl/src/test/resources/bad.cwl b/cwl/src/test/resources/bad.cwl deleted file mode 100644 index d38ed298731..00000000000 --- a/cwl/src/test/resources/bad.cwl +++ /dev/null @@ -1 +0,0 @@ -gibberish diff --git a/cwl/src/test/resources/bad2.cwl b/cwl/src/test/resources/bad2.cwl deleted file mode 100644 index d38ed298731..00000000000 --- a/cwl/src/test/resources/bad2.cwl +++ /dev/null @@ -1 +0,0 @@ -gibberish diff --git a/cwl/src/test/resources/brokenlinks.cwl b/cwl/src/test/resources/brokenlinks.cwl deleted file mode 100644 index c710358ce61..00000000000 --- a/cwl/src/test/resources/brokenlinks.cwl +++ /dev/null @@ -1,23 +0,0 @@ -cwlVersion: v1.0 -class: Workflow -inputs: - inp: File - ex: string -outputs: - classout: - type: File - outputSource: compile/classfile - -steps: - untar: - run: wrong.cwl - in: - tarfile: inp - extractfile: ex - out: [example_out] - - compile: - run: wrong2.cwl - in: - src: untar/example_out - out: [classfile] diff --git a/cwl/src/test/resources/cwl/lodash.js b/cwl/src/test/resources/cwl/lodash.js deleted file mode 100644 index 9b95dfefe87..00000000000 --- a/cwl/src/test/resources/cwl/lodash.js +++ /dev/null @@ -1,17112 +0,0 @@ -/** - * @license - * Lodash - * Copyright OpenJS Foundation and other contributors - * Released under MIT license - * Based on Underscore.js 1.8.3 - * Copyright Jeremy Ashkenas, DocumentCloud and Investigative Reporters & Editors - */ -;(function() { - - /** Used as a safe reference for `undefined` in pre-ES5 environments. */ - var undefined; - - /** Used as the semantic version number. */ - var VERSION = '4.17.15'; - - /** Used as the size to enable large array optimizations. */ - var LARGE_ARRAY_SIZE = 200; - - /** Error message constants. */ - var CORE_ERROR_TEXT = 'Unsupported core-js use. Try https://npms.io/search?q=ponyfill.', - FUNC_ERROR_TEXT = 'Expected a function'; - - /** Used to stand-in for `undefined` hash values. */ - var HASH_UNDEFINED = '__lodash_hash_undefined__'; - - /** Used as the maximum memoize cache size. */ - var MAX_MEMOIZE_SIZE = 500; - - /** Used as the internal argument placeholder. */ - var PLACEHOLDER = '__lodash_placeholder__'; - - /** Used to compose bitmasks for cloning. */ - var CLONE_DEEP_FLAG = 1, - CLONE_FLAT_FLAG = 2, - CLONE_SYMBOLS_FLAG = 4; - - /** Used to compose bitmasks for value comparisons. */ - var COMPARE_PARTIAL_FLAG = 1, - COMPARE_UNORDERED_FLAG = 2; - - /** Used to compose bitmasks for function metadata. */ - var WRAP_BIND_FLAG = 1, - WRAP_BIND_KEY_FLAG = 2, - WRAP_CURRY_BOUND_FLAG = 4, - WRAP_CURRY_FLAG = 8, - WRAP_CURRY_RIGHT_FLAG = 16, - WRAP_PARTIAL_FLAG = 32, - WRAP_PARTIAL_RIGHT_FLAG = 64, - WRAP_ARY_FLAG = 128, - WRAP_REARG_FLAG = 256, - WRAP_FLIP_FLAG = 512; - - /** Used as default options for `_.truncate`. */ - var DEFAULT_TRUNC_LENGTH = 30, - DEFAULT_TRUNC_OMISSION = '...'; - - /** Used to detect hot functions by number of calls within a span of milliseconds. */ - var HOT_COUNT = 800, - HOT_SPAN = 16; - - /** Used to indicate the type of lazy iteratees. */ - var LAZY_FILTER_FLAG = 1, - LAZY_MAP_FLAG = 2, - LAZY_WHILE_FLAG = 3; - - /** Used as references for various `Number` constants. */ - var INFINITY = 1 / 0, - MAX_SAFE_INTEGER = 9007199254740991, - MAX_INTEGER = 1.7976931348623157e+308, - NAN = 0 / 0; - - /** Used as references for the maximum length and index of an array. */ - var MAX_ARRAY_LENGTH = 4294967295, - MAX_ARRAY_INDEX = MAX_ARRAY_LENGTH - 1, - HALF_MAX_ARRAY_LENGTH = MAX_ARRAY_LENGTH >>> 1; - - /** Used to associate wrap methods with their bit flags. */ - var wrapFlags = [ - ['ary', WRAP_ARY_FLAG], - ['bind', WRAP_BIND_FLAG], - ['bindKey', WRAP_BIND_KEY_FLAG], - ['curry', WRAP_CURRY_FLAG], - ['curryRight', WRAP_CURRY_RIGHT_FLAG], - ['flip', WRAP_FLIP_FLAG], - ['partial', WRAP_PARTIAL_FLAG], - ['partialRight', WRAP_PARTIAL_RIGHT_FLAG], - ['rearg', WRAP_REARG_FLAG] - ]; - - /** `Object#toString` result references. */ - var argsTag = '[object Arguments]', - arrayTag = '[object Array]', - asyncTag = '[object AsyncFunction]', - boolTag = '[object Boolean]', - dateTag = '[object Date]', - domExcTag = '[object DOMException]', - errorTag = '[object Error]', - funcTag = '[object Function]', - genTag = '[object GeneratorFunction]', - mapTag = '[object Map]', - numberTag = '[object Number]', - nullTag = '[object Null]', - objectTag = '[object Object]', - promiseTag = '[object Promise]', - proxyTag = '[object Proxy]', - regexpTag = '[object RegExp]', - setTag = '[object Set]', - stringTag = '[object String]', - symbolTag = '[object Symbol]', - undefinedTag = '[object Undefined]', - weakMapTag = '[object WeakMap]', - weakSetTag = '[object WeakSet]'; - - var arrayBufferTag = '[object ArrayBuffer]', - dataViewTag = '[object DataView]', - float32Tag = '[object Float32Array]', - float64Tag = '[object Float64Array]', - int8Tag = '[object Int8Array]', - int16Tag = '[object Int16Array]', - int32Tag = '[object Int32Array]', - uint8Tag = '[object Uint8Array]', - uint8ClampedTag = '[object Uint8ClampedArray]', - uint16Tag = '[object Uint16Array]', - uint32Tag = '[object Uint32Array]'; - - /** Used to match empty string literals in compiled template source. */ - var reEmptyStringLeading = /\b__p \+= '';/g, - reEmptyStringMiddle = /\b(__p \+=) '' \+/g, - reEmptyStringTrailing = /(__e\(.*?\)|\b__t\)) \+\n'';/g; - - /** Used to match HTML entities and HTML characters. */ - var reEscapedHtml = /&(?:amp|lt|gt|quot|#39);/g, - reUnescapedHtml = /[&<>"']/g, - reHasEscapedHtml = RegExp(reEscapedHtml.source), - reHasUnescapedHtml = RegExp(reUnescapedHtml.source); - - /** Used to match template delimiters. */ - var reEscape = /<%-([\s\S]+?)%>/g, - reEvaluate = /<%([\s\S]+?)%>/g, - reInterpolate = /<%=([\s\S]+?)%>/g; - - /** Used to match property names within property paths. */ - var reIsDeepProp = /\.|\[(?:[^[\]]*|(["'])(?:(?!\1)[^\\]|\\.)*?\1)\]/, - reIsPlainProp = /^\w*$/, - rePropName = /[^.[\]]+|\[(?:(-?\d+(?:\.\d+)?)|(["'])((?:(?!\2)[^\\]|\\.)*?)\2)\]|(?=(?:\.|\[\])(?:\.|\[\]|$))/g; - - /** - * Used to match `RegExp` - * [syntax characters](http://ecma-international.org/ecma-262/7.0/#sec-patterns). - */ - var reRegExpChar = /[\\^$.*+?()[\]{}|]/g, - reHasRegExpChar = RegExp(reRegExpChar.source); - - /** Used to match leading and trailing whitespace. */ - var reTrim = /^\s+|\s+$/g, - reTrimStart = /^\s+/, - reTrimEnd = /\s+$/; - - /** Used to match wrap detail comments. */ - var reWrapComment = /\{(?:\n\/\* \[wrapped with .+\] \*\/)?\n?/, - reWrapDetails = /\{\n\/\* \[wrapped with (.+)\] \*/, - reSplitDetails = /,? & /; - - /** Used to match words composed of alphanumeric characters. */ - var reAsciiWord = /[^\x00-\x2f\x3a-\x40\x5b-\x60\x7b-\x7f]+/g; - - /** Used to match backslashes in property paths. */ - var reEscapeChar = /\\(\\)?/g; - - /** - * Used to match - * [ES template delimiters](http://ecma-international.org/ecma-262/7.0/#sec-template-literal-lexical-components). - */ - var reEsTemplate = /\$\{([^\\}]*(?:\\.[^\\}]*)*)\}/g; - - /** Used to match `RegExp` flags from their coerced string values. */ - var reFlags = /\w*$/; - - /** Used to detect bad signed hexadecimal string values. */ - var reIsBadHex = /^[-+]0x[0-9a-f]+$/i; - - /** Used to detect binary string values. */ - var reIsBinary = /^0b[01]+$/i; - - /** Used to detect host constructors (Safari). */ - var reIsHostCtor = /^\[object .+?Constructor\]$/; - - /** Used to detect octal string values. */ - var reIsOctal = /^0o[0-7]+$/i; - - /** Used to detect unsigned integer values. */ - var reIsUint = /^(?:0|[1-9]\d*)$/; - - /** Used to match Latin Unicode letters (excluding mathematical operators). */ - var reLatin = /[\xc0-\xd6\xd8-\xf6\xf8-\xff\u0100-\u017f]/g; - - /** Used to ensure capturing order of template delimiters. */ - var reNoMatch = /($^)/; - - /** Used to match unescaped characters in compiled string literals. */ - var reUnescapedString = /['\n\r\u2028\u2029\\]/g; - - /** Used to compose unicode character classes. */ - var rsAstralRange = '\\ud800-\\udfff', - rsComboMarksRange = '\\u0300-\\u036f', - reComboHalfMarksRange = '\\ufe20-\\ufe2f', - rsComboSymbolsRange = '\\u20d0-\\u20ff', - rsComboRange = rsComboMarksRange + reComboHalfMarksRange + rsComboSymbolsRange, - rsDingbatRange = '\\u2700-\\u27bf', - rsLowerRange = 'a-z\\xdf-\\xf6\\xf8-\\xff', - rsMathOpRange = '\\xac\\xb1\\xd7\\xf7', - rsNonCharRange = '\\x00-\\x2f\\x3a-\\x40\\x5b-\\x60\\x7b-\\xbf', - rsPunctuationRange = '\\u2000-\\u206f', - rsSpaceRange = ' \\t\\x0b\\f\\xa0\\ufeff\\n\\r\\u2028\\u2029\\u1680\\u180e\\u2000\\u2001\\u2002\\u2003\\u2004\\u2005\\u2006\\u2007\\u2008\\u2009\\u200a\\u202f\\u205f\\u3000', - rsUpperRange = 'A-Z\\xc0-\\xd6\\xd8-\\xde', - rsVarRange = '\\ufe0e\\ufe0f', - rsBreakRange = rsMathOpRange + rsNonCharRange + rsPunctuationRange + rsSpaceRange; - - /** Used to compose unicode capture groups. */ - var rsApos = "['\u2019]", - rsAstral = '[' + rsAstralRange + ']', - rsBreak = '[' + rsBreakRange + ']', - rsCombo = '[' + rsComboRange + ']', - rsDigits = '\\d+', - rsDingbat = '[' + rsDingbatRange + ']', - rsLower = '[' + rsLowerRange + ']', - rsMisc = '[^' + rsAstralRange + rsBreakRange + rsDigits + rsDingbatRange + rsLowerRange + rsUpperRange + ']', - rsFitz = '\\ud83c[\\udffb-\\udfff]', - rsModifier = '(?:' + rsCombo + '|' + rsFitz + ')', - rsNonAstral = '[^' + rsAstralRange + ']', - rsRegional = '(?:\\ud83c[\\udde6-\\uddff]){2}', - rsSurrPair = '[\\ud800-\\udbff][\\udc00-\\udfff]', - rsUpper = '[' + rsUpperRange + ']', - rsZWJ = '\\u200d'; - - /** Used to compose unicode regexes. */ - var rsMiscLower = '(?:' + rsLower + '|' + rsMisc + ')', - rsMiscUpper = '(?:' + rsUpper + '|' + rsMisc + ')', - rsOptContrLower = '(?:' + rsApos + '(?:d|ll|m|re|s|t|ve))?', - rsOptContrUpper = '(?:' + rsApos + '(?:D|LL|M|RE|S|T|VE))?', - reOptMod = rsModifier + '?', - rsOptVar = '[' + rsVarRange + ']?', - rsOptJoin = '(?:' + rsZWJ + '(?:' + [rsNonAstral, rsRegional, rsSurrPair].join('|') + ')' + rsOptVar + reOptMod + ')*', - rsOrdLower = '\\d*(?:1st|2nd|3rd|(?![123])\\dth)(?=\\b|[A-Z_])', - rsOrdUpper = '\\d*(?:1ST|2ND|3RD|(?![123])\\dTH)(?=\\b|[a-z_])', - rsSeq = rsOptVar + reOptMod + rsOptJoin, - rsEmoji = '(?:' + [rsDingbat, rsRegional, rsSurrPair].join('|') + ')' + rsSeq, - rsSymbol = '(?:' + [rsNonAstral + rsCombo + '?', rsCombo, rsRegional, rsSurrPair, rsAstral].join('|') + ')'; - - /** Used to match apostrophes. */ - var reApos = RegExp(rsApos, 'g'); - - /** - * Used to match [combining diacritical marks](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks) and - * [combining diacritical marks for symbols](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks_for_Symbols). - */ - var reComboMark = RegExp(rsCombo, 'g'); - - /** Used to match [string symbols](https://mathiasbynens.be/notes/javascript-unicode). */ - var reUnicode = RegExp(rsFitz + '(?=' + rsFitz + ')|' + rsSymbol + rsSeq, 'g'); - - /** Used to match complex or compound words. */ - var reUnicodeWord = RegExp([ - rsUpper + '?' + rsLower + '+' + rsOptContrLower + '(?=' + [rsBreak, rsUpper, '$'].join('|') + ')', - rsMiscUpper + '+' + rsOptContrUpper + '(?=' + [rsBreak, rsUpper + rsMiscLower, '$'].join('|') + ')', - rsUpper + '?' + rsMiscLower + '+' + rsOptContrLower, - rsUpper + '+' + rsOptContrUpper, - rsOrdUpper, - rsOrdLower, - rsDigits, - rsEmoji - ].join('|'), 'g'); - - /** Used to detect strings with [zero-width joiners or code points from the astral planes](http://eev.ee/blog/2015/09/12/dark-corners-of-unicode/). */ - var reHasUnicode = RegExp('[' + rsZWJ + rsAstralRange + rsComboRange + rsVarRange + ']'); - - /** Used to detect strings that need a more robust regexp to match words. */ - var reHasUnicodeWord = /[a-z][A-Z]|[A-Z]{2}[a-z]|[0-9][a-zA-Z]|[a-zA-Z][0-9]|[^a-zA-Z0-9 ]/; - - /** Used to assign default `context` object properties. */ - var contextProps = [ - 'Array', 'Buffer', 'DataView', 'Date', 'Error', 'Float32Array', 'Float64Array', - 'Function', 'Int8Array', 'Int16Array', 'Int32Array', 'Map', 'Math', 'Object', - 'Promise', 'RegExp', 'Set', 'String', 'Symbol', 'TypeError', 'Uint8Array', - 'Uint8ClampedArray', 'Uint16Array', 'Uint32Array', 'WeakMap', - '_', 'clearTimeout', 'isFinite', 'parseInt', 'setTimeout' - ]; - - /** Used to make template sourceURLs easier to identify. */ - var templateCounter = -1; - - /** Used to identify `toStringTag` values of typed arrays. */ - var typedArrayTags = {}; - typedArrayTags[float32Tag] = typedArrayTags[float64Tag] = - typedArrayTags[int8Tag] = typedArrayTags[int16Tag] = - typedArrayTags[int32Tag] = typedArrayTags[uint8Tag] = - typedArrayTags[uint8ClampedTag] = typedArrayTags[uint16Tag] = - typedArrayTags[uint32Tag] = true; - typedArrayTags[argsTag] = typedArrayTags[arrayTag] = - typedArrayTags[arrayBufferTag] = typedArrayTags[boolTag] = - typedArrayTags[dataViewTag] = typedArrayTags[dateTag] = - typedArrayTags[errorTag] = typedArrayTags[funcTag] = - typedArrayTags[mapTag] = typedArrayTags[numberTag] = - typedArrayTags[objectTag] = typedArrayTags[regexpTag] = - typedArrayTags[setTag] = typedArrayTags[stringTag] = - typedArrayTags[weakMapTag] = false; - - /** Used to identify `toStringTag` values supported by `_.clone`. */ - var cloneableTags = {}; - cloneableTags[argsTag] = cloneableTags[arrayTag] = - cloneableTags[arrayBufferTag] = cloneableTags[dataViewTag] = - cloneableTags[boolTag] = cloneableTags[dateTag] = - cloneableTags[float32Tag] = cloneableTags[float64Tag] = - cloneableTags[int8Tag] = cloneableTags[int16Tag] = - cloneableTags[int32Tag] = cloneableTags[mapTag] = - cloneableTags[numberTag] = cloneableTags[objectTag] = - cloneableTags[regexpTag] = cloneableTags[setTag] = - cloneableTags[stringTag] = cloneableTags[symbolTag] = - cloneableTags[uint8Tag] = cloneableTags[uint8ClampedTag] = - cloneableTags[uint16Tag] = cloneableTags[uint32Tag] = true; - cloneableTags[errorTag] = cloneableTags[funcTag] = - cloneableTags[weakMapTag] = false; - - /** Used to map Latin Unicode letters to basic Latin letters. */ - var deburredLetters = { - // Latin-1 Supplement block. - '\xc0': 'A', '\xc1': 'A', '\xc2': 'A', '\xc3': 'A', '\xc4': 'A', '\xc5': 'A', - '\xe0': 'a', '\xe1': 'a', '\xe2': 'a', '\xe3': 'a', '\xe4': 'a', '\xe5': 'a', - '\xc7': 'C', '\xe7': 'c', - '\xd0': 'D', '\xf0': 'd', - '\xc8': 'E', '\xc9': 'E', '\xca': 'E', '\xcb': 'E', - '\xe8': 'e', '\xe9': 'e', '\xea': 'e', '\xeb': 'e', - '\xcc': 'I', '\xcd': 'I', '\xce': 'I', '\xcf': 'I', - '\xec': 'i', '\xed': 'i', '\xee': 'i', '\xef': 'i', - '\xd1': 'N', '\xf1': 'n', - '\xd2': 'O', '\xd3': 'O', '\xd4': 'O', '\xd5': 'O', '\xd6': 'O', '\xd8': 'O', - '\xf2': 'o', '\xf3': 'o', '\xf4': 'o', '\xf5': 'o', '\xf6': 'o', '\xf8': 'o', - '\xd9': 'U', '\xda': 'U', '\xdb': 'U', '\xdc': 'U', - '\xf9': 'u', '\xfa': 'u', '\xfb': 'u', '\xfc': 'u', - '\xdd': 'Y', '\xfd': 'y', '\xff': 'y', - '\xc6': 'Ae', '\xe6': 'ae', - '\xde': 'Th', '\xfe': 'th', - '\xdf': 'ss', - // Latin Extended-A block. - '\u0100': 'A', '\u0102': 'A', '\u0104': 'A', - '\u0101': 'a', '\u0103': 'a', '\u0105': 'a', - '\u0106': 'C', '\u0108': 'C', '\u010a': 'C', '\u010c': 'C', - '\u0107': 'c', '\u0109': 'c', '\u010b': 'c', '\u010d': 'c', - '\u010e': 'D', '\u0110': 'D', '\u010f': 'd', '\u0111': 'd', - '\u0112': 'E', '\u0114': 'E', '\u0116': 'E', '\u0118': 'E', '\u011a': 'E', - '\u0113': 'e', '\u0115': 'e', '\u0117': 'e', '\u0119': 'e', '\u011b': 'e', - '\u011c': 'G', '\u011e': 'G', '\u0120': 'G', '\u0122': 'G', - '\u011d': 'g', '\u011f': 'g', '\u0121': 'g', '\u0123': 'g', - '\u0124': 'H', '\u0126': 'H', '\u0125': 'h', '\u0127': 'h', - '\u0128': 'I', '\u012a': 'I', '\u012c': 'I', '\u012e': 'I', '\u0130': 'I', - '\u0129': 'i', '\u012b': 'i', '\u012d': 'i', '\u012f': 'i', '\u0131': 'i', - '\u0134': 'J', '\u0135': 'j', - '\u0136': 'K', '\u0137': 'k', '\u0138': 'k', - '\u0139': 'L', '\u013b': 'L', '\u013d': 'L', '\u013f': 'L', '\u0141': 'L', - '\u013a': 'l', '\u013c': 'l', '\u013e': 'l', '\u0140': 'l', '\u0142': 'l', - '\u0143': 'N', '\u0145': 'N', '\u0147': 'N', '\u014a': 'N', - '\u0144': 'n', '\u0146': 'n', '\u0148': 'n', '\u014b': 'n', - '\u014c': 'O', '\u014e': 'O', '\u0150': 'O', - '\u014d': 'o', '\u014f': 'o', '\u0151': 'o', - '\u0154': 'R', '\u0156': 'R', '\u0158': 'R', - '\u0155': 'r', '\u0157': 'r', '\u0159': 'r', - '\u015a': 'S', '\u015c': 'S', '\u015e': 'S', '\u0160': 'S', - '\u015b': 's', '\u015d': 's', '\u015f': 's', '\u0161': 's', - '\u0162': 'T', '\u0164': 'T', '\u0166': 'T', - '\u0163': 't', '\u0165': 't', '\u0167': 't', - '\u0168': 'U', '\u016a': 'U', '\u016c': 'U', '\u016e': 'U', '\u0170': 'U', '\u0172': 'U', - '\u0169': 'u', '\u016b': 'u', '\u016d': 'u', '\u016f': 'u', '\u0171': 'u', '\u0173': 'u', - '\u0174': 'W', '\u0175': 'w', - '\u0176': 'Y', '\u0177': 'y', '\u0178': 'Y', - '\u0179': 'Z', '\u017b': 'Z', '\u017d': 'Z', - '\u017a': 'z', '\u017c': 'z', '\u017e': 'z', - '\u0132': 'IJ', '\u0133': 'ij', - '\u0152': 'Oe', '\u0153': 'oe', - '\u0149': "'n", '\u017f': 's' - }; - - /** Used to map characters to HTML entities. */ - var htmlEscapes = { - '&': '&', - '<': '<', - '>': '>', - '"': '"', - "'": ''' - }; - - /** Used to map HTML entities to characters. */ - var htmlUnescapes = { - '&': '&', - '<': '<', - '>': '>', - '"': '"', - ''': "'" - }; - - /** Used to escape characters for inclusion in compiled string literals. */ - var stringEscapes = { - '\\': '\\', - "'": "'", - '\n': 'n', - '\r': 'r', - '\u2028': 'u2028', - '\u2029': 'u2029' - }; - - /** Built-in method references without a dependency on `root`. */ - var freeParseFloat = parseFloat, - freeParseInt = parseInt; - - /** Detect free variable `global` from Node.js. */ - var freeGlobal = typeof global == 'object' && global && global.Object === Object && global; - - /** Detect free variable `self`. */ - var freeSelf = typeof self == 'object' && self && self.Object === Object && self; - - /** Used as a reference to the global object. */ - var root = freeGlobal || freeSelf || Function('return this')(); - - /** Detect free variable `exports`. */ - var freeExports = typeof exports == 'object' && exports && !exports.nodeType && exports; - - /** Detect free variable `module`. */ - var freeModule = freeExports && typeof module == 'object' && module && !module.nodeType && module; - - /** Detect the popular CommonJS extension `module.exports`. */ - var moduleExports = freeModule && freeModule.exports === freeExports; - - /** Detect free variable `process` from Node.js. */ - var freeProcess = moduleExports && freeGlobal.process; - - /** Used to access faster Node.js helpers. */ - var nodeUtil = (function() { - try { - // Use `util.types` for Node.js 10+. - var types = freeModule && freeModule.require && freeModule.require('util').types; - - if (types) { - return types; - } - - // Legacy `process.binding('util')` for Node.js < 10. - return freeProcess && freeProcess.binding && freeProcess.binding('util'); - } catch (e) {} - }()); - - /* Node.js helper references. */ - var nodeIsArrayBuffer = nodeUtil && nodeUtil.isArrayBuffer, - nodeIsDate = nodeUtil && nodeUtil.isDate, - nodeIsMap = nodeUtil && nodeUtil.isMap, - nodeIsRegExp = nodeUtil && nodeUtil.isRegExp, - nodeIsSet = nodeUtil && nodeUtil.isSet, - nodeIsTypedArray = nodeUtil && nodeUtil.isTypedArray; - - /*--------------------------------------------------------------------------*/ - - /** - * A faster alternative to `Function#apply`, this function invokes `func` - * with the `this` binding of `thisArg` and the arguments of `args`. - * - * @private - * @param {Function} func The function to invoke. - * @param {*} thisArg The `this` binding of `func`. - * @param {Array} args The arguments to invoke `func` with. - * @returns {*} Returns the result of `func`. - */ - function apply(func, thisArg, args) { - switch (args.length) { - case 0: return func.call(thisArg); - case 1: return func.call(thisArg, args[0]); - case 2: return func.call(thisArg, args[0], args[1]); - case 3: return func.call(thisArg, args[0], args[1], args[2]); - } - return func.apply(thisArg, args); - } - - /** - * A specialized version of `baseAggregator` for arrays. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} setter The function to set `accumulator` values. - * @param {Function} iteratee The iteratee to transform keys. - * @param {Object} accumulator The initial aggregated object. - * @returns {Function} Returns `accumulator`. - */ - function arrayAggregator(array, setter, iteratee, accumulator) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - var value = array[index]; - setter(accumulator, value, iteratee(value), array); - } - return accumulator; - } - - /** - * A specialized version of `_.forEach` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns `array`. - */ - function arrayEach(array, iteratee) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - if (iteratee(array[index], index, array) === false) { - break; - } - } - return array; - } - - /** - * A specialized version of `_.forEachRight` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns `array`. - */ - function arrayEachRight(array, iteratee) { - var length = array == null ? 0 : array.length; - - while (length--) { - if (iteratee(array[length], length, array) === false) { - break; - } - } - return array; - } - - /** - * A specialized version of `_.every` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if all elements pass the predicate check, - * else `false`. - */ - function arrayEvery(array, predicate) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - if (!predicate(array[index], index, array)) { - return false; - } - } - return true; - } - - /** - * A specialized version of `_.filter` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {Array} Returns the new filtered array. - */ - function arrayFilter(array, predicate) { - var index = -1, - length = array == null ? 0 : array.length, - resIndex = 0, - result = []; - - while (++index < length) { - var value = array[index]; - if (predicate(value, index, array)) { - result[resIndex++] = value; - } - } - return result; - } - - /** - * A specialized version of `_.includes` for arrays without support for - * specifying an index to search from. - * - * @private - * @param {Array} [array] The array to inspect. - * @param {*} target The value to search for. - * @returns {boolean} Returns `true` if `target` is found, else `false`. - */ - function arrayIncludes(array, value) { - var length = array == null ? 0 : array.length; - return !!length && baseIndexOf(array, value, 0) > -1; - } - - /** - * This function is like `arrayIncludes` except that it accepts a comparator. - * - * @private - * @param {Array} [array] The array to inspect. - * @param {*} target The value to search for. - * @param {Function} comparator The comparator invoked per element. - * @returns {boolean} Returns `true` if `target` is found, else `false`. - */ - function arrayIncludesWith(array, value, comparator) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - if (comparator(value, array[index])) { - return true; - } - } - return false; - } - - /** - * A specialized version of `_.map` for arrays without support for iteratee - * shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns the new mapped array. - */ - function arrayMap(array, iteratee) { - var index = -1, - length = array == null ? 0 : array.length, - result = Array(length); - - while (++index < length) { - result[index] = iteratee(array[index], index, array); - } - return result; - } - - /** - * Appends the elements of `values` to `array`. - * - * @private - * @param {Array} array The array to modify. - * @param {Array} values The values to append. - * @returns {Array} Returns `array`. - */ - function arrayPush(array, values) { - var index = -1, - length = values.length, - offset = array.length; - - while (++index < length) { - array[offset + index] = values[index]; - } - return array; - } - - /** - * A specialized version of `_.reduce` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @param {boolean} [initAccum] Specify using the first element of `array` as - * the initial value. - * @returns {*} Returns the accumulated value. - */ - function arrayReduce(array, iteratee, accumulator, initAccum) { - var index = -1, - length = array == null ? 0 : array.length; - - if (initAccum && length) { - accumulator = array[++index]; - } - while (++index < length) { - accumulator = iteratee(accumulator, array[index], index, array); - } - return accumulator; - } - - /** - * A specialized version of `_.reduceRight` for arrays without support for - * iteratee shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @param {boolean} [initAccum] Specify using the last element of `array` as - * the initial value. - * @returns {*} Returns the accumulated value. - */ - function arrayReduceRight(array, iteratee, accumulator, initAccum) { - var length = array == null ? 0 : array.length; - if (initAccum && length) { - accumulator = array[--length]; - } - while (length--) { - accumulator = iteratee(accumulator, array[length], length, array); - } - return accumulator; - } - - /** - * A specialized version of `_.some` for arrays without support for iteratee - * shorthands. - * - * @private - * @param {Array} [array] The array to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if any element passes the predicate check, - * else `false`. - */ - function arraySome(array, predicate) { - var index = -1, - length = array == null ? 0 : array.length; - - while (++index < length) { - if (predicate(array[index], index, array)) { - return true; - } - } - return false; - } - - /** - * Gets the size of an ASCII `string`. - * - * @private - * @param {string} string The string inspect. - * @returns {number} Returns the string size. - */ - var asciiSize = baseProperty('length'); - - /** - * Converts an ASCII `string` to an array. - * - * @private - * @param {string} string The string to convert. - * @returns {Array} Returns the converted array. - */ - function asciiToArray(string) { - return string.split(''); - } - - /** - * Splits an ASCII `string` into an array of its words. - * - * @private - * @param {string} The string to inspect. - * @returns {Array} Returns the words of `string`. - */ - function asciiWords(string) { - return string.match(reAsciiWord) || []; - } - - /** - * The base implementation of methods like `_.findKey` and `_.findLastKey`, - * without support for iteratee shorthands, which iterates over `collection` - * using `eachFunc`. - * - * @private - * @param {Array|Object} collection The collection to inspect. - * @param {Function} predicate The function invoked per iteration. - * @param {Function} eachFunc The function to iterate over `collection`. - * @returns {*} Returns the found element or its key, else `undefined`. - */ - function baseFindKey(collection, predicate, eachFunc) { - var result; - eachFunc(collection, function(value, key, collection) { - if (predicate(value, key, collection)) { - result = key; - return false; - } - }); - return result; - } - - /** - * The base implementation of `_.findIndex` and `_.findLastIndex` without - * support for iteratee shorthands. - * - * @private - * @param {Array} array The array to inspect. - * @param {Function} predicate The function invoked per iteration. - * @param {number} fromIndex The index to search from. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function baseFindIndex(array, predicate, fromIndex, fromRight) { - var length = array.length, - index = fromIndex + (fromRight ? 1 : -1); - - while ((fromRight ? index-- : ++index < length)) { - if (predicate(array[index], index, array)) { - return index; - } - } - return -1; - } - - /** - * The base implementation of `_.indexOf` without `fromIndex` bounds checks. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} fromIndex The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function baseIndexOf(array, value, fromIndex) { - return value === value - ? strictIndexOf(array, value, fromIndex) - : baseFindIndex(array, baseIsNaN, fromIndex); - } - - /** - * This function is like `baseIndexOf` except that it accepts a comparator. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} fromIndex The index to search from. - * @param {Function} comparator The comparator invoked per element. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function baseIndexOfWith(array, value, fromIndex, comparator) { - var index = fromIndex - 1, - length = array.length; - - while (++index < length) { - if (comparator(array[index], value)) { - return index; - } - } - return -1; - } - - /** - * The base implementation of `_.isNaN` without support for number objects. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `NaN`, else `false`. - */ - function baseIsNaN(value) { - return value !== value; - } - - /** - * The base implementation of `_.mean` and `_.meanBy` without support for - * iteratee shorthands. - * - * @private - * @param {Array} array The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {number} Returns the mean. - */ - function baseMean(array, iteratee) { - var length = array == null ? 0 : array.length; - return length ? (baseSum(array, iteratee) / length) : NAN; - } - - /** - * The base implementation of `_.property` without support for deep paths. - * - * @private - * @param {string} key The key of the property to get. - * @returns {Function} Returns the new accessor function. - */ - function baseProperty(key) { - return function(object) { - return object == null ? undefined : object[key]; - }; - } - - /** - * The base implementation of `_.propertyOf` without support for deep paths. - * - * @private - * @param {Object} object The object to query. - * @returns {Function} Returns the new accessor function. - */ - function basePropertyOf(object) { - return function(key) { - return object == null ? undefined : object[key]; - }; - } - - /** - * The base implementation of `_.reduce` and `_.reduceRight`, without support - * for iteratee shorthands, which iterates over `collection` using `eachFunc`. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {*} accumulator The initial value. - * @param {boolean} initAccum Specify using the first or last element of - * `collection` as the initial value. - * @param {Function} eachFunc The function to iterate over `collection`. - * @returns {*} Returns the accumulated value. - */ - function baseReduce(collection, iteratee, accumulator, initAccum, eachFunc) { - eachFunc(collection, function(value, index, collection) { - accumulator = initAccum - ? (initAccum = false, value) - : iteratee(accumulator, value, index, collection); - }); - return accumulator; - } - - /** - * The base implementation of `_.sortBy` which uses `comparer` to define the - * sort order of `array` and replaces criteria objects with their corresponding - * values. - * - * @private - * @param {Array} array The array to sort. - * @param {Function} comparer The function to define sort order. - * @returns {Array} Returns `array`. - */ - function baseSortBy(array, comparer) { - var length = array.length; - - array.sort(comparer); - while (length--) { - array[length] = array[length].value; - } - return array; - } - - /** - * The base implementation of `_.sum` and `_.sumBy` without support for - * iteratee shorthands. - * - * @private - * @param {Array} array The array to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {number} Returns the sum. - */ - function baseSum(array, iteratee) { - var result, - index = -1, - length = array.length; - - while (++index < length) { - var current = iteratee(array[index]); - if (current !== undefined) { - result = result === undefined ? current : (result + current); - } - } - return result; - } - - /** - * The base implementation of `_.times` without support for iteratee shorthands - * or max array length checks. - * - * @private - * @param {number} n The number of times to invoke `iteratee`. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns the array of results. - */ - function baseTimes(n, iteratee) { - var index = -1, - result = Array(n); - - while (++index < n) { - result[index] = iteratee(index); - } - return result; - } - - /** - * The base implementation of `_.toPairs` and `_.toPairsIn` which creates an array - * of key-value pairs for `object` corresponding to the property names of `props`. - * - * @private - * @param {Object} object The object to query. - * @param {Array} props The property names to get values for. - * @returns {Object} Returns the key-value pairs. - */ - function baseToPairs(object, props) { - return arrayMap(props, function(key) { - return [key, object[key]]; - }); - } - - /** - * The base implementation of `_.unary` without support for storing metadata. - * - * @private - * @param {Function} func The function to cap arguments for. - * @returns {Function} Returns the new capped function. - */ - function baseUnary(func) { - return function(value) { - return func(value); - }; - } - - /** - * The base implementation of `_.values` and `_.valuesIn` which creates an - * array of `object` property values corresponding to the property names - * of `props`. - * - * @private - * @param {Object} object The object to query. - * @param {Array} props The property names to get values for. - * @returns {Object} Returns the array of property values. - */ - function baseValues(object, props) { - return arrayMap(props, function(key) { - return object[key]; - }); - } - - /** - * Checks if a `cache` value for `key` exists. - * - * @private - * @param {Object} cache The cache to query. - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. - */ - function cacheHas(cache, key) { - return cache.has(key); - } - - /** - * Used by `_.trim` and `_.trimStart` to get the index of the first string symbol - * that is not found in the character symbols. - * - * @private - * @param {Array} strSymbols The string symbols to inspect. - * @param {Array} chrSymbols The character symbols to find. - * @returns {number} Returns the index of the first unmatched string symbol. - */ - function charsStartIndex(strSymbols, chrSymbols) { - var index = -1, - length = strSymbols.length; - - while (++index < length && baseIndexOf(chrSymbols, strSymbols[index], 0) > -1) {} - return index; - } - - /** - * Used by `_.trim` and `_.trimEnd` to get the index of the last string symbol - * that is not found in the character symbols. - * - * @private - * @param {Array} strSymbols The string symbols to inspect. - * @param {Array} chrSymbols The character symbols to find. - * @returns {number} Returns the index of the last unmatched string symbol. - */ - function charsEndIndex(strSymbols, chrSymbols) { - var index = strSymbols.length; - - while (index-- && baseIndexOf(chrSymbols, strSymbols[index], 0) > -1) {} - return index; - } - - /** - * Gets the number of `placeholder` occurrences in `array`. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} placeholder The placeholder to search for. - * @returns {number} Returns the placeholder count. - */ - function countHolders(array, placeholder) { - var length = array.length, - result = 0; - - while (length--) { - if (array[length] === placeholder) { - ++result; - } - } - return result; - } - - /** - * Used by `_.deburr` to convert Latin-1 Supplement and Latin Extended-A - * letters to basic Latin letters. - * - * @private - * @param {string} letter The matched letter to deburr. - * @returns {string} Returns the deburred letter. - */ - var deburrLetter = basePropertyOf(deburredLetters); - - /** - * Used by `_.escape` to convert characters to HTML entities. - * - * @private - * @param {string} chr The matched character to escape. - * @returns {string} Returns the escaped character. - */ - var escapeHtmlChar = basePropertyOf(htmlEscapes); - - /** - * Used by `_.template` to escape characters for inclusion in compiled string literals. - * - * @private - * @param {string} chr The matched character to escape. - * @returns {string} Returns the escaped character. - */ - function escapeStringChar(chr) { - return '\\' + stringEscapes[chr]; - } - - /** - * Gets the value at `key` of `object`. - * - * @private - * @param {Object} [object] The object to query. - * @param {string} key The key of the property to get. - * @returns {*} Returns the property value. - */ - function getValue(object, key) { - return object == null ? undefined : object[key]; - } - - /** - * Checks if `string` contains Unicode symbols. - * - * @private - * @param {string} string The string to inspect. - * @returns {boolean} Returns `true` if a symbol is found, else `false`. - */ - function hasUnicode(string) { - return reHasUnicode.test(string); - } - - /** - * Checks if `string` contains a word composed of Unicode symbols. - * - * @private - * @param {string} string The string to inspect. - * @returns {boolean} Returns `true` if a word is found, else `false`. - */ - function hasUnicodeWord(string) { - return reHasUnicodeWord.test(string); - } - - /** - * Converts `iterator` to an array. - * - * @private - * @param {Object} iterator The iterator to convert. - * @returns {Array} Returns the converted array. - */ - function iteratorToArray(iterator) { - var data, - result = []; - - while (!(data = iterator.next()).done) { - result.push(data.value); - } - return result; - } - - /** - * Converts `map` to its key-value pairs. - * - * @private - * @param {Object} map The map to convert. - * @returns {Array} Returns the key-value pairs. - */ - function mapToArray(map) { - var index = -1, - result = Array(map.size); - - map.forEach(function(value, key) { - result[++index] = [key, value]; - }); - return result; - } - - /** - * Creates a unary function that invokes `func` with its argument transformed. - * - * @private - * @param {Function} func The function to wrap. - * @param {Function} transform The argument transform. - * @returns {Function} Returns the new function. - */ - function overArg(func, transform) { - return function(arg) { - return func(transform(arg)); - }; - } - - /** - * Replaces all `placeholder` elements in `array` with an internal placeholder - * and returns an array of their indexes. - * - * @private - * @param {Array} array The array to modify. - * @param {*} placeholder The placeholder to replace. - * @returns {Array} Returns the new array of placeholder indexes. - */ - function replaceHolders(array, placeholder) { - var index = -1, - length = array.length, - resIndex = 0, - result = []; - - while (++index < length) { - var value = array[index]; - if (value === placeholder || value === PLACEHOLDER) { - array[index] = PLACEHOLDER; - result[resIndex++] = index; - } - } - return result; - } - - /** - * Converts `set` to an array of its values. - * - * @private - * @param {Object} set The set to convert. - * @returns {Array} Returns the values. - */ - function setToArray(set) { - var index = -1, - result = Array(set.size); - - set.forEach(function(value) { - result[++index] = value; - }); - return result; - } - - /** - * Converts `set` to its value-value pairs. - * - * @private - * @param {Object} set The set to convert. - * @returns {Array} Returns the value-value pairs. - */ - function setToPairs(set) { - var index = -1, - result = Array(set.size); - - set.forEach(function(value) { - result[++index] = [value, value]; - }); - return result; - } - - /** - * A specialized version of `_.indexOf` which performs strict equality - * comparisons of values, i.e. `===`. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} fromIndex The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function strictIndexOf(array, value, fromIndex) { - var index = fromIndex - 1, - length = array.length; - - while (++index < length) { - if (array[index] === value) { - return index; - } - } - return -1; - } - - /** - * A specialized version of `_.lastIndexOf` which performs strict equality - * comparisons of values, i.e. `===`. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} fromIndex The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function strictLastIndexOf(array, value, fromIndex) { - var index = fromIndex + 1; - while (index--) { - if (array[index] === value) { - return index; - } - } - return index; - } - - /** - * Gets the number of symbols in `string`. - * - * @private - * @param {string} string The string to inspect. - * @returns {number} Returns the string size. - */ - function stringSize(string) { - return hasUnicode(string) - ? unicodeSize(string) - : asciiSize(string); - } - - /** - * Converts `string` to an array. - * - * @private - * @param {string} string The string to convert. - * @returns {Array} Returns the converted array. - */ - function stringToArray(string) { - return hasUnicode(string) - ? unicodeToArray(string) - : asciiToArray(string); - } - - /** - * Used by `_.unescape` to convert HTML entities to characters. - * - * @private - * @param {string} chr The matched character to unescape. - * @returns {string} Returns the unescaped character. - */ - var unescapeHtmlChar = basePropertyOf(htmlUnescapes); - - /** - * Gets the size of a Unicode `string`. - * - * @private - * @param {string} string The string inspect. - * @returns {number} Returns the string size. - */ - function unicodeSize(string) { - var result = reUnicode.lastIndex = 0; - while (reUnicode.test(string)) { - ++result; - } - return result; - } - - /** - * Converts a Unicode `string` to an array. - * - * @private - * @param {string} string The string to convert. - * @returns {Array} Returns the converted array. - */ - function unicodeToArray(string) { - return string.match(reUnicode) || []; - } - - /** - * Splits a Unicode `string` into an array of its words. - * - * @private - * @param {string} The string to inspect. - * @returns {Array} Returns the words of `string`. - */ - function unicodeWords(string) { - return string.match(reUnicodeWord) || []; - } - - /*--------------------------------------------------------------------------*/ - - /** - * Create a new pristine `lodash` function using the `context` object. - * - * @static - * @memberOf _ - * @since 1.1.0 - * @category Util - * @param {Object} [context=root] The context object. - * @returns {Function} Returns a new `lodash` function. - * @example - * - * _.mixin({ 'foo': _.constant('foo') }); - * - * var lodash = _.runInContext(); - * lodash.mixin({ 'bar': lodash.constant('bar') }); - * - * _.isFunction(_.foo); - * // => true - * _.isFunction(_.bar); - * // => false - * - * lodash.isFunction(lodash.foo); - * // => false - * lodash.isFunction(lodash.bar); - * // => true - * - * // Create a suped-up `defer` in Node.js. - * var defer = _.runInContext({ 'setTimeout': setImmediate }).defer; - */ - var runInContext = (function runInContext(context) { - context = context == null ? root : _.defaults(root.Object(), context, _.pick(root, contextProps)); - - /** Built-in constructor references. */ - var Array = context.Array, - Date = context.Date, - Error = context.Error, - Function = context.Function, - Math = context.Math, - Object = context.Object, - RegExp = context.RegExp, - String = context.String, - TypeError = context.TypeError; - - /** Used for built-in method references. */ - var arrayProto = Array.prototype, - funcProto = Function.prototype, - objectProto = Object.prototype; - - /** Used to detect overreaching core-js shims. */ - var coreJsData = context['__core-js_shared__']; - - /** Used to resolve the decompiled source of functions. */ - var funcToString = funcProto.toString; - - /** Used to check objects for own properties. */ - var hasOwnProperty = objectProto.hasOwnProperty; - - /** Used to generate unique IDs. */ - var idCounter = 0; - - /** Used to detect methods masquerading as native. */ - var maskSrcKey = (function() { - var uid = /[^.]+$/.exec(coreJsData && coreJsData.keys && coreJsData.keys.IE_PROTO || ''); - return uid ? ('Symbol(src)_1.' + uid) : ''; - }()); - - /** - * Used to resolve the - * [`toStringTag`](http://ecma-international.org/ecma-262/7.0/#sec-object.prototype.tostring) - * of values. - */ - var nativeObjectToString = objectProto.toString; - - /** Used to infer the `Object` constructor. */ - var objectCtorString = funcToString.call(Object); - - /** Used to restore the original `_` reference in `_.noConflict`. */ - var oldDash = root._; - - /** Used to detect if a method is native. */ - var reIsNative = RegExp('^' + - funcToString.call(hasOwnProperty).replace(reRegExpChar, '\\$&') - .replace(/hasOwnProperty|(function).*?(?=\\\()| for .+?(?=\\\])/g, '$1.*?') + '$' - ); - - /** Built-in value references. */ - var Buffer = moduleExports ? context.Buffer : undefined, - Symbol = context.Symbol, - Uint8Array = context.Uint8Array, - allocUnsafe = Buffer ? Buffer.allocUnsafe : undefined, - getPrototype = overArg(Object.getPrototypeOf, Object), - objectCreate = Object.create, - propertyIsEnumerable = objectProto.propertyIsEnumerable, - splice = arrayProto.splice, - spreadableSymbol = Symbol ? Symbol.isConcatSpreadable : undefined, - symIterator = Symbol ? Symbol.iterator : undefined, - symToStringTag = Symbol ? Symbol.toStringTag : undefined; - - var defineProperty = (function() { - try { - var func = getNative(Object, 'defineProperty'); - func({}, '', {}); - return func; - } catch (e) {} - }()); - - /** Mocked built-ins. */ - var ctxClearTimeout = context.clearTimeout !== root.clearTimeout && context.clearTimeout, - ctxNow = Date && Date.now !== root.Date.now && Date.now, - ctxSetTimeout = context.setTimeout !== root.setTimeout && context.setTimeout; - - /* Built-in method references for those with the same name as other `lodash` methods. */ - var nativeCeil = Math.ceil, - nativeFloor = Math.floor, - nativeGetSymbols = Object.getOwnPropertySymbols, - nativeIsBuffer = Buffer ? Buffer.isBuffer : undefined, - nativeIsFinite = context.isFinite, - nativeJoin = arrayProto.join, - nativeKeys = overArg(Object.keys, Object), - nativeMax = Math.max, - nativeMin = Math.min, - nativeNow = Date.now, - nativeParseInt = context.parseInt, - nativeRandom = Math.random, - nativeReverse = arrayProto.reverse; - - /* Built-in method references that are verified to be native. */ - var DataView = getNative(context, 'DataView'), - Map = getNative(context, 'Map'), - Promise = getNative(context, 'Promise'), - Set = getNative(context, 'Set'), - WeakMap = getNative(context, 'WeakMap'), - nativeCreate = getNative(Object, 'create'); - - /** Used to store function metadata. */ - var metaMap = WeakMap && new WeakMap; - - /** Used to lookup unminified function names. */ - var realNames = {}; - - /** Used to detect maps, sets, and weakmaps. */ - var dataViewCtorString = toSource(DataView), - mapCtorString = toSource(Map), - promiseCtorString = toSource(Promise), - setCtorString = toSource(Set), - weakMapCtorString = toSource(WeakMap); - - /** Used to convert symbols to primitives and strings. */ - var symbolProto = Symbol ? Symbol.prototype : undefined, - symbolValueOf = symbolProto ? symbolProto.valueOf : undefined, - symbolToString = symbolProto ? symbolProto.toString : undefined; - - /*------------------------------------------------------------------------*/ - - /** - * Creates a `lodash` object which wraps `value` to enable implicit method - * chain sequences. Methods that operate on and return arrays, collections, - * and functions can be chained together. Methods that retrieve a single value - * or may return a primitive value will automatically end the chain sequence - * and return the unwrapped value. Otherwise, the value must be unwrapped - * with `_#value`. - * - * Explicit chain sequences, which must be unwrapped with `_#value`, may be - * enabled using `_.chain`. - * - * The execution of chained methods is lazy, that is, it's deferred until - * `_#value` is implicitly or explicitly called. - * - * Lazy evaluation allows several methods to support shortcut fusion. - * Shortcut fusion is an optimization to merge iteratee calls; this avoids - * the creation of intermediate arrays and can greatly reduce the number of - * iteratee executions. Sections of a chain sequence qualify for shortcut - * fusion if the section is applied to an array and iteratees accept only - * one argument. The heuristic for whether a section qualifies for shortcut - * fusion is subject to change. - * - * Chaining is supported in custom builds as long as the `_#value` method is - * directly or indirectly included in the build. - * - * In addition to lodash methods, wrappers have `Array` and `String` methods. - * - * The wrapper `Array` methods are: - * `concat`, `join`, `pop`, `push`, `shift`, `sort`, `splice`, and `unshift` - * - * The wrapper `String` methods are: - * `replace` and `split` - * - * The wrapper methods that support shortcut fusion are: - * `at`, `compact`, `drop`, `dropRight`, `dropWhile`, `filter`, `find`, - * `findLast`, `head`, `initial`, `last`, `map`, `reject`, `reverse`, `slice`, - * `tail`, `take`, `takeRight`, `takeRightWhile`, `takeWhile`, and `toArray` - * - * The chainable wrapper methods are: - * `after`, `ary`, `assign`, `assignIn`, `assignInWith`, `assignWith`, `at`, - * `before`, `bind`, `bindAll`, `bindKey`, `castArray`, `chain`, `chunk`, - * `commit`, `compact`, `concat`, `conforms`, `constant`, `countBy`, `create`, - * `curry`, `debounce`, `defaults`, `defaultsDeep`, `defer`, `delay`, - * `difference`, `differenceBy`, `differenceWith`, `drop`, `dropRight`, - * `dropRightWhile`, `dropWhile`, `extend`, `extendWith`, `fill`, `filter`, - * `flatMap`, `flatMapDeep`, `flatMapDepth`, `flatten`, `flattenDeep`, - * `flattenDepth`, `flip`, `flow`, `flowRight`, `fromPairs`, `functions`, - * `functionsIn`, `groupBy`, `initial`, `intersection`, `intersectionBy`, - * `intersectionWith`, `invert`, `invertBy`, `invokeMap`, `iteratee`, `keyBy`, - * `keys`, `keysIn`, `map`, `mapKeys`, `mapValues`, `matches`, `matchesProperty`, - * `memoize`, `merge`, `mergeWith`, `method`, `methodOf`, `mixin`, `negate`, - * `nthArg`, `omit`, `omitBy`, `once`, `orderBy`, `over`, `overArgs`, - * `overEvery`, `overSome`, `partial`, `partialRight`, `partition`, `pick`, - * `pickBy`, `plant`, `property`, `propertyOf`, `pull`, `pullAll`, `pullAllBy`, - * `pullAllWith`, `pullAt`, `push`, `range`, `rangeRight`, `rearg`, `reject`, - * `remove`, `rest`, `reverse`, `sampleSize`, `set`, `setWith`, `shuffle`, - * `slice`, `sort`, `sortBy`, `splice`, `spread`, `tail`, `take`, `takeRight`, - * `takeRightWhile`, `takeWhile`, `tap`, `throttle`, `thru`, `toArray`, - * `toPairs`, `toPairsIn`, `toPath`, `toPlainObject`, `transform`, `unary`, - * `union`, `unionBy`, `unionWith`, `uniq`, `uniqBy`, `uniqWith`, `unset`, - * `unshift`, `unzip`, `unzipWith`, `update`, `updateWith`, `values`, - * `valuesIn`, `without`, `wrap`, `xor`, `xorBy`, `xorWith`, `zip`, - * `zipObject`, `zipObjectDeep`, and `zipWith` - * - * The wrapper methods that are **not** chainable by default are: - * `add`, `attempt`, `camelCase`, `capitalize`, `ceil`, `clamp`, `clone`, - * `cloneDeep`, `cloneDeepWith`, `cloneWith`, `conformsTo`, `deburr`, - * `defaultTo`, `divide`, `each`, `eachRight`, `endsWith`, `eq`, `escape`, - * `escapeRegExp`, `every`, `find`, `findIndex`, `findKey`, `findLast`, - * `findLastIndex`, `findLastKey`, `first`, `floor`, `forEach`, `forEachRight`, - * `forIn`, `forInRight`, `forOwn`, `forOwnRight`, `get`, `gt`, `gte`, `has`, - * `hasIn`, `head`, `identity`, `includes`, `indexOf`, `inRange`, `invoke`, - * `isArguments`, `isArray`, `isArrayBuffer`, `isArrayLike`, `isArrayLikeObject`, - * `isBoolean`, `isBuffer`, `isDate`, `isElement`, `isEmpty`, `isEqual`, - * `isEqualWith`, `isError`, `isFinite`, `isFunction`, `isInteger`, `isLength`, - * `isMap`, `isMatch`, `isMatchWith`, `isNaN`, `isNative`, `isNil`, `isNull`, - * `isNumber`, `isObject`, `isObjectLike`, `isPlainObject`, `isRegExp`, - * `isSafeInteger`, `isSet`, `isString`, `isUndefined`, `isTypedArray`, - * `isWeakMap`, `isWeakSet`, `join`, `kebabCase`, `last`, `lastIndexOf`, - * `lowerCase`, `lowerFirst`, `lt`, `lte`, `max`, `maxBy`, `mean`, `meanBy`, - * `min`, `minBy`, `multiply`, `noConflict`, `noop`, `now`, `nth`, `pad`, - * `padEnd`, `padStart`, `parseInt`, `pop`, `random`, `reduce`, `reduceRight`, - * `repeat`, `result`, `round`, `runInContext`, `sample`, `shift`, `size`, - * `snakeCase`, `some`, `sortedIndex`, `sortedIndexBy`, `sortedLastIndex`, - * `sortedLastIndexBy`, `startCase`, `startsWith`, `stubArray`, `stubFalse`, - * `stubObject`, `stubString`, `stubTrue`, `subtract`, `sum`, `sumBy`, - * `template`, `times`, `toFinite`, `toInteger`, `toJSON`, `toLength`, - * `toLower`, `toNumber`, `toSafeInteger`, `toString`, `toUpper`, `trim`, - * `trimEnd`, `trimStart`, `truncate`, `unescape`, `uniqueId`, `upperCase`, - * `upperFirst`, `value`, and `words` - * - * @name _ - * @constructor - * @category Seq - * @param {*} value The value to wrap in a `lodash` instance. - * @returns {Object} Returns the new `lodash` wrapper instance. - * @example - * - * function square(n) { - * return n * n; - * } - * - * var wrapped = _([1, 2, 3]); - * - * // Returns an unwrapped value. - * wrapped.reduce(_.add); - * // => 6 - * - * // Returns a wrapped value. - * var squares = wrapped.map(square); - * - * _.isArray(squares); - * // => false - * - * _.isArray(squares.value()); - * // => true - */ - function lodash(value) { - if (isObjectLike(value) && !isArray(value) && !(value instanceof LazyWrapper)) { - if (value instanceof LodashWrapper) { - return value; - } - if (hasOwnProperty.call(value, '__wrapped__')) { - return wrapperClone(value); - } - } - return new LodashWrapper(value); - } - - /** - * The base implementation of `_.create` without support for assigning - * properties to the created object. - * - * @private - * @param {Object} proto The object to inherit from. - * @returns {Object} Returns the new object. - */ - var baseCreate = (function() { - function object() {} - return function(proto) { - if (!isObject(proto)) { - return {}; - } - if (objectCreate) { - return objectCreate(proto); - } - object.prototype = proto; - var result = new object; - object.prototype = undefined; - return result; - }; - }()); - - /** - * The function whose prototype chain sequence wrappers inherit from. - * - * @private - */ - function baseLodash() { - // No operation performed. - } - - /** - * The base constructor for creating `lodash` wrapper objects. - * - * @private - * @param {*} value The value to wrap. - * @param {boolean} [chainAll] Enable explicit method chain sequences. - */ - function LodashWrapper(value, chainAll) { - this.__wrapped__ = value; - this.__actions__ = []; - this.__chain__ = !!chainAll; - this.__index__ = 0; - this.__values__ = undefined; - } - - /** - * By default, the template delimiters used by lodash are like those in - * embedded Ruby (ERB) as well as ES2015 template strings. Change the - * following template settings to use alternative delimiters. - * - * @static - * @memberOf _ - * @type {Object} - */ - lodash.templateSettings = { - - /** - * Used to detect `data` property values to be HTML-escaped. - * - * @memberOf _.templateSettings - * @type {RegExp} - */ - 'escape': reEscape, - - /** - * Used to detect code to be evaluated. - * - * @memberOf _.templateSettings - * @type {RegExp} - */ - 'evaluate': reEvaluate, - - /** - * Used to detect `data` property values to inject. - * - * @memberOf _.templateSettings - * @type {RegExp} - */ - 'interpolate': reInterpolate, - - /** - * Used to reference the data object in the template text. - * - * @memberOf _.templateSettings - * @type {string} - */ - 'variable': '', - - /** - * Used to import variables into the compiled template. - * - * @memberOf _.templateSettings - * @type {Object} - */ - 'imports': { - - /** - * A reference to the `lodash` function. - * - * @memberOf _.templateSettings.imports - * @type {Function} - */ - '_': lodash - } - }; - - // Ensure wrappers are instances of `baseLodash`. - lodash.prototype = baseLodash.prototype; - lodash.prototype.constructor = lodash; - - LodashWrapper.prototype = baseCreate(baseLodash.prototype); - LodashWrapper.prototype.constructor = LodashWrapper; - - /*------------------------------------------------------------------------*/ - - /** - * Creates a lazy wrapper object which wraps `value` to enable lazy evaluation. - * - * @private - * @constructor - * @param {*} value The value to wrap. - */ - function LazyWrapper(value) { - this.__wrapped__ = value; - this.__actions__ = []; - this.__dir__ = 1; - this.__filtered__ = false; - this.__iteratees__ = []; - this.__takeCount__ = MAX_ARRAY_LENGTH; - this.__views__ = []; - } - - /** - * Creates a clone of the lazy wrapper object. - * - * @private - * @name clone - * @memberOf LazyWrapper - * @returns {Object} Returns the cloned `LazyWrapper` object. - */ - function lazyClone() { - var result = new LazyWrapper(this.__wrapped__); - result.__actions__ = copyArray(this.__actions__); - result.__dir__ = this.__dir__; - result.__filtered__ = this.__filtered__; - result.__iteratees__ = copyArray(this.__iteratees__); - result.__takeCount__ = this.__takeCount__; - result.__views__ = copyArray(this.__views__); - return result; - } - - /** - * Reverses the direction of lazy iteration. - * - * @private - * @name reverse - * @memberOf LazyWrapper - * @returns {Object} Returns the new reversed `LazyWrapper` object. - */ - function lazyReverse() { - if (this.__filtered__) { - var result = new LazyWrapper(this); - result.__dir__ = -1; - result.__filtered__ = true; - } else { - result = this.clone(); - result.__dir__ *= -1; - } - return result; - } - - /** - * Extracts the unwrapped value from its lazy wrapper. - * - * @private - * @name value - * @memberOf LazyWrapper - * @returns {*} Returns the unwrapped value. - */ - function lazyValue() { - var array = this.__wrapped__.value(), - dir = this.__dir__, - isArr = isArray(array), - isRight = dir < 0, - arrLength = isArr ? array.length : 0, - view = getView(0, arrLength, this.__views__), - start = view.start, - end = view.end, - length = end - start, - index = isRight ? end : (start - 1), - iteratees = this.__iteratees__, - iterLength = iteratees.length, - resIndex = 0, - takeCount = nativeMin(length, this.__takeCount__); - - if (!isArr || (!isRight && arrLength == length && takeCount == length)) { - return baseWrapperValue(array, this.__actions__); - } - var result = []; - - outer: - while (length-- && resIndex < takeCount) { - index += dir; - - var iterIndex = -1, - value = array[index]; - - while (++iterIndex < iterLength) { - var data = iteratees[iterIndex], - iteratee = data.iteratee, - type = data.type, - computed = iteratee(value); - - if (type == LAZY_MAP_FLAG) { - value = computed; - } else if (!computed) { - if (type == LAZY_FILTER_FLAG) { - continue outer; - } else { - break outer; - } - } - } - result[resIndex++] = value; - } - return result; - } - - // Ensure `LazyWrapper` is an instance of `baseLodash`. - LazyWrapper.prototype = baseCreate(baseLodash.prototype); - LazyWrapper.prototype.constructor = LazyWrapper; - - /*------------------------------------------------------------------------*/ - - /** - * Creates a hash object. - * - * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. - */ - function Hash(entries) { - var index = -1, - length = entries == null ? 0 : entries.length; - - this.clear(); - while (++index < length) { - var entry = entries[index]; - this.set(entry[0], entry[1]); - } - } - - /** - * Removes all key-value entries from the hash. - * - * @private - * @name clear - * @memberOf Hash - */ - function hashClear() { - this.__data__ = nativeCreate ? nativeCreate(null) : {}; - this.size = 0; - } - - /** - * Removes `key` and its value from the hash. - * - * @private - * @name delete - * @memberOf Hash - * @param {Object} hash The hash to modify. - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. - */ - function hashDelete(key) { - var result = this.has(key) && delete this.__data__[key]; - this.size -= result ? 1 : 0; - return result; - } - - /** - * Gets the hash value for `key`. - * - * @private - * @name get - * @memberOf Hash - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. - */ - function hashGet(key) { - var data = this.__data__; - if (nativeCreate) { - var result = data[key]; - return result === HASH_UNDEFINED ? undefined : result; - } - return hasOwnProperty.call(data, key) ? data[key] : undefined; - } - - /** - * Checks if a hash value for `key` exists. - * - * @private - * @name has - * @memberOf Hash - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. - */ - function hashHas(key) { - var data = this.__data__; - return nativeCreate ? (data[key] !== undefined) : hasOwnProperty.call(data, key); - } - - /** - * Sets the hash `key` to `value`. - * - * @private - * @name set - * @memberOf Hash - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the hash instance. - */ - function hashSet(key, value) { - var data = this.__data__; - this.size += this.has(key) ? 0 : 1; - data[key] = (nativeCreate && value === undefined) ? HASH_UNDEFINED : value; - return this; - } - - // Add methods to `Hash`. - Hash.prototype.clear = hashClear; - Hash.prototype['delete'] = hashDelete; - Hash.prototype.get = hashGet; - Hash.prototype.has = hashHas; - Hash.prototype.set = hashSet; - - /*------------------------------------------------------------------------*/ - - /** - * Creates an list cache object. - * - * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. - */ - function ListCache(entries) { - var index = -1, - length = entries == null ? 0 : entries.length; - - this.clear(); - while (++index < length) { - var entry = entries[index]; - this.set(entry[0], entry[1]); - } - } - - /** - * Removes all key-value entries from the list cache. - * - * @private - * @name clear - * @memberOf ListCache - */ - function listCacheClear() { - this.__data__ = []; - this.size = 0; - } - - /** - * Removes `key` and its value from the list cache. - * - * @private - * @name delete - * @memberOf ListCache - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. - */ - function listCacheDelete(key) { - var data = this.__data__, - index = assocIndexOf(data, key); - - if (index < 0) { - return false; - } - var lastIndex = data.length - 1; - if (index == lastIndex) { - data.pop(); - } else { - splice.call(data, index, 1); - } - --this.size; - return true; - } - - /** - * Gets the list cache value for `key`. - * - * @private - * @name get - * @memberOf ListCache - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. - */ - function listCacheGet(key) { - var data = this.__data__, - index = assocIndexOf(data, key); - - return index < 0 ? undefined : data[index][1]; - } - - /** - * Checks if a list cache value for `key` exists. - * - * @private - * @name has - * @memberOf ListCache - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. - */ - function listCacheHas(key) { - return assocIndexOf(this.__data__, key) > -1; - } - - /** - * Sets the list cache `key` to `value`. - * - * @private - * @name set - * @memberOf ListCache - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the list cache instance. - */ - function listCacheSet(key, value) { - var data = this.__data__, - index = assocIndexOf(data, key); - - if (index < 0) { - ++this.size; - data.push([key, value]); - } else { - data[index][1] = value; - } - return this; - } - - // Add methods to `ListCache`. - ListCache.prototype.clear = listCacheClear; - ListCache.prototype['delete'] = listCacheDelete; - ListCache.prototype.get = listCacheGet; - ListCache.prototype.has = listCacheHas; - ListCache.prototype.set = listCacheSet; - - /*------------------------------------------------------------------------*/ - - /** - * Creates a map cache object to store key-value pairs. - * - * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. - */ - function MapCache(entries) { - var index = -1, - length = entries == null ? 0 : entries.length; - - this.clear(); - while (++index < length) { - var entry = entries[index]; - this.set(entry[0], entry[1]); - } - } - - /** - * Removes all key-value entries from the map. - * - * @private - * @name clear - * @memberOf MapCache - */ - function mapCacheClear() { - this.size = 0; - this.__data__ = { - 'hash': new Hash, - 'map': new (Map || ListCache), - 'string': new Hash - }; - } - - /** - * Removes `key` and its value from the map. - * - * @private - * @name delete - * @memberOf MapCache - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. - */ - function mapCacheDelete(key) { - var result = getMapData(this, key)['delete'](key); - this.size -= result ? 1 : 0; - return result; - } - - /** - * Gets the map value for `key`. - * - * @private - * @name get - * @memberOf MapCache - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. - */ - function mapCacheGet(key) { - return getMapData(this, key).get(key); - } - - /** - * Checks if a map value for `key` exists. - * - * @private - * @name has - * @memberOf MapCache - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. - */ - function mapCacheHas(key) { - return getMapData(this, key).has(key); - } - - /** - * Sets the map `key` to `value`. - * - * @private - * @name set - * @memberOf MapCache - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the map cache instance. - */ - function mapCacheSet(key, value) { - var data = getMapData(this, key), - size = data.size; - - data.set(key, value); - this.size += data.size == size ? 0 : 1; - return this; - } - - // Add methods to `MapCache`. - MapCache.prototype.clear = mapCacheClear; - MapCache.prototype['delete'] = mapCacheDelete; - MapCache.prototype.get = mapCacheGet; - MapCache.prototype.has = mapCacheHas; - MapCache.prototype.set = mapCacheSet; - - /*------------------------------------------------------------------------*/ - - /** - * - * Creates an array cache object to store unique values. - * - * @private - * @constructor - * @param {Array} [values] The values to cache. - */ - function SetCache(values) { - var index = -1, - length = values == null ? 0 : values.length; - - this.__data__ = new MapCache; - while (++index < length) { - this.add(values[index]); - } - } - - /** - * Adds `value` to the array cache. - * - * @private - * @name add - * @memberOf SetCache - * @alias push - * @param {*} value The value to cache. - * @returns {Object} Returns the cache instance. - */ - function setCacheAdd(value) { - this.__data__.set(value, HASH_UNDEFINED); - return this; - } - - /** - * Checks if `value` is in the array cache. - * - * @private - * @name has - * @memberOf SetCache - * @param {*} value The value to search for. - * @returns {number} Returns `true` if `value` is found, else `false`. - */ - function setCacheHas(value) { - return this.__data__.has(value); - } - - // Add methods to `SetCache`. - SetCache.prototype.add = SetCache.prototype.push = setCacheAdd; - SetCache.prototype.has = setCacheHas; - - /*------------------------------------------------------------------------*/ - - /** - * Creates a stack cache object to store key-value pairs. - * - * @private - * @constructor - * @param {Array} [entries] The key-value pairs to cache. - */ - function Stack(entries) { - var data = this.__data__ = new ListCache(entries); - this.size = data.size; - } - - /** - * Removes all key-value entries from the stack. - * - * @private - * @name clear - * @memberOf Stack - */ - function stackClear() { - this.__data__ = new ListCache; - this.size = 0; - } - - /** - * Removes `key` and its value from the stack. - * - * @private - * @name delete - * @memberOf Stack - * @param {string} key The key of the value to remove. - * @returns {boolean} Returns `true` if the entry was removed, else `false`. - */ - function stackDelete(key) { - var data = this.__data__, - result = data['delete'](key); - - this.size = data.size; - return result; - } - - /** - * Gets the stack value for `key`. - * - * @private - * @name get - * @memberOf Stack - * @param {string} key The key of the value to get. - * @returns {*} Returns the entry value. - */ - function stackGet(key) { - return this.__data__.get(key); - } - - /** - * Checks if a stack value for `key` exists. - * - * @private - * @name has - * @memberOf Stack - * @param {string} key The key of the entry to check. - * @returns {boolean} Returns `true` if an entry for `key` exists, else `false`. - */ - function stackHas(key) { - return this.__data__.has(key); - } - - /** - * Sets the stack `key` to `value`. - * - * @private - * @name set - * @memberOf Stack - * @param {string} key The key of the value to set. - * @param {*} value The value to set. - * @returns {Object} Returns the stack cache instance. - */ - function stackSet(key, value) { - var data = this.__data__; - if (data instanceof ListCache) { - var pairs = data.__data__; - if (!Map || (pairs.length < LARGE_ARRAY_SIZE - 1)) { - pairs.push([key, value]); - this.size = ++data.size; - return this; - } - data = this.__data__ = new MapCache(pairs); - } - data.set(key, value); - this.size = data.size; - return this; - } - - // Add methods to `Stack`. - Stack.prototype.clear = stackClear; - Stack.prototype['delete'] = stackDelete; - Stack.prototype.get = stackGet; - Stack.prototype.has = stackHas; - Stack.prototype.set = stackSet; - - /*------------------------------------------------------------------------*/ - - /** - * Creates an array of the enumerable property names of the array-like `value`. - * - * @private - * @param {*} value The value to query. - * @param {boolean} inherited Specify returning inherited property names. - * @returns {Array} Returns the array of property names. - */ - function arrayLikeKeys(value, inherited) { - var isArr = isArray(value), - isArg = !isArr && isArguments(value), - isBuff = !isArr && !isArg && isBuffer(value), - isType = !isArr && !isArg && !isBuff && isTypedArray(value), - skipIndexes = isArr || isArg || isBuff || isType, - result = skipIndexes ? baseTimes(value.length, String) : [], - length = result.length; - - for (var key in value) { - if ((inherited || hasOwnProperty.call(value, key)) && - !(skipIndexes && ( - // Safari 9 has enumerable `arguments.length` in strict mode. - key == 'length' || - // Node.js 0.10 has enumerable non-index properties on buffers. - (isBuff && (key == 'offset' || key == 'parent')) || - // PhantomJS 2 has enumerable non-index properties on typed arrays. - (isType && (key == 'buffer' || key == 'byteLength' || key == 'byteOffset')) || - // Skip index properties. - isIndex(key, length) - ))) { - result.push(key); - } - } - return result; - } - - /** - * A specialized version of `_.sample` for arrays. - * - * @private - * @param {Array} array The array to sample. - * @returns {*} Returns the random element. - */ - function arraySample(array) { - var length = array.length; - return length ? array[baseRandom(0, length - 1)] : undefined; - } - - /** - * A specialized version of `_.sampleSize` for arrays. - * - * @private - * @param {Array} array The array to sample. - * @param {number} n The number of elements to sample. - * @returns {Array} Returns the random elements. - */ - function arraySampleSize(array, n) { - return shuffleSelf(copyArray(array), baseClamp(n, 0, array.length)); - } - - /** - * A specialized version of `_.shuffle` for arrays. - * - * @private - * @param {Array} array The array to shuffle. - * @returns {Array} Returns the new shuffled array. - */ - function arrayShuffle(array) { - return shuffleSelf(copyArray(array)); - } - - /** - * This function is like `assignValue` except that it doesn't assign - * `undefined` values. - * - * @private - * @param {Object} object The object to modify. - * @param {string} key The key of the property to assign. - * @param {*} value The value to assign. - */ - function assignMergeValue(object, key, value) { - if ((value !== undefined && !eq(object[key], value)) || - (value === undefined && !(key in object))) { - baseAssignValue(object, key, value); - } - } - - /** - * Assigns `value` to `key` of `object` if the existing value is not equivalent - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. - * - * @private - * @param {Object} object The object to modify. - * @param {string} key The key of the property to assign. - * @param {*} value The value to assign. - */ - function assignValue(object, key, value) { - var objValue = object[key]; - if (!(hasOwnProperty.call(object, key) && eq(objValue, value)) || - (value === undefined && !(key in object))) { - baseAssignValue(object, key, value); - } - } - - /** - * Gets the index at which the `key` is found in `array` of key-value pairs. - * - * @private - * @param {Array} array The array to inspect. - * @param {*} key The key to search for. - * @returns {number} Returns the index of the matched value, else `-1`. - */ - function assocIndexOf(array, key) { - var length = array.length; - while (length--) { - if (eq(array[length][0], key)) { - return length; - } - } - return -1; - } - - /** - * Aggregates elements of `collection` on `accumulator` with keys transformed - * by `iteratee` and values set by `setter`. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} setter The function to set `accumulator` values. - * @param {Function} iteratee The iteratee to transform keys. - * @param {Object} accumulator The initial aggregated object. - * @returns {Function} Returns `accumulator`. - */ - function baseAggregator(collection, setter, iteratee, accumulator) { - baseEach(collection, function(value, key, collection) { - setter(accumulator, value, iteratee(value), collection); - }); - return accumulator; - } - - /** - * The base implementation of `_.assign` without support for multiple sources - * or `customizer` functions. - * - * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @returns {Object} Returns `object`. - */ - function baseAssign(object, source) { - return object && copyObject(source, keys(source), object); - } - - /** - * The base implementation of `_.assignIn` without support for multiple sources - * or `customizer` functions. - * - * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @returns {Object} Returns `object`. - */ - function baseAssignIn(object, source) { - return object && copyObject(source, keysIn(source), object); - } - - /** - * The base implementation of `assignValue` and `assignMergeValue` without - * value checks. - * - * @private - * @param {Object} object The object to modify. - * @param {string} key The key of the property to assign. - * @param {*} value The value to assign. - */ - function baseAssignValue(object, key, value) { - if (key == '__proto__' && defineProperty) { - defineProperty(object, key, { - 'configurable': true, - 'enumerable': true, - 'value': value, - 'writable': true - }); - } else { - object[key] = value; - } - } - - /** - * The base implementation of `_.at` without support for individual paths. - * - * @private - * @param {Object} object The object to iterate over. - * @param {string[]} paths The property paths to pick. - * @returns {Array} Returns the picked elements. - */ - function baseAt(object, paths) { - var index = -1, - length = paths.length, - result = Array(length), - skip = object == null; - - while (++index < length) { - result[index] = skip ? undefined : get(object, paths[index]); - } - return result; - } - - /** - * The base implementation of `_.clamp` which doesn't coerce arguments. - * - * @private - * @param {number} number The number to clamp. - * @param {number} [lower] The lower bound. - * @param {number} upper The upper bound. - * @returns {number} Returns the clamped number. - */ - function baseClamp(number, lower, upper) { - if (number === number) { - if (upper !== undefined) { - number = number <= upper ? number : upper; - } - if (lower !== undefined) { - number = number >= lower ? number : lower; - } - } - return number; - } - - /** - * The base implementation of `_.clone` and `_.cloneDeep` which tracks - * traversed objects. - * - * @private - * @param {*} value The value to clone. - * @param {boolean} bitmask The bitmask flags. - * 1 - Deep clone - * 2 - Flatten inherited properties - * 4 - Clone symbols - * @param {Function} [customizer] The function to customize cloning. - * @param {string} [key] The key of `value`. - * @param {Object} [object] The parent object of `value`. - * @param {Object} [stack] Tracks traversed objects and their clone counterparts. - * @returns {*} Returns the cloned value. - */ - function baseClone(value, bitmask, customizer, key, object, stack) { - var result, - isDeep = bitmask & CLONE_DEEP_FLAG, - isFlat = bitmask & CLONE_FLAT_FLAG, - isFull = bitmask & CLONE_SYMBOLS_FLAG; - - if (customizer) { - result = object ? customizer(value, key, object, stack) : customizer(value); - } - if (result !== undefined) { - return result; - } - if (!isObject(value)) { - return value; - } - var isArr = isArray(value); - if (isArr) { - result = initCloneArray(value); - if (!isDeep) { - return copyArray(value, result); - } - } else { - var tag = getTag(value), - isFunc = tag == funcTag || tag == genTag; - - if (isBuffer(value)) { - return cloneBuffer(value, isDeep); - } - if (tag == objectTag || tag == argsTag || (isFunc && !object)) { - result = (isFlat || isFunc) ? {} : initCloneObject(value); - if (!isDeep) { - return isFlat - ? copySymbolsIn(value, baseAssignIn(result, value)) - : copySymbols(value, baseAssign(result, value)); - } - } else { - if (!cloneableTags[tag]) { - return object ? value : {}; - } - result = initCloneByTag(value, tag, isDeep); - } - } - // Check for circular references and return its corresponding clone. - stack || (stack = new Stack); - var stacked = stack.get(value); - if (stacked) { - return stacked; - } - stack.set(value, result); - - if (isSet(value)) { - value.forEach(function(subValue) { - result.add(baseClone(subValue, bitmask, customizer, subValue, value, stack)); - }); - } else if (isMap(value)) { - value.forEach(function(subValue, key) { - result.set(key, baseClone(subValue, bitmask, customizer, key, value, stack)); - }); - } - - var keysFunc = isFull - ? (isFlat ? getAllKeysIn : getAllKeys) - : (isFlat ? keysIn : keys); - - var props = isArr ? undefined : keysFunc(value); - arrayEach(props || value, function(subValue, key) { - if (props) { - key = subValue; - subValue = value[key]; - } - // Recursively populate clone (susceptible to call stack limits). - assignValue(result, key, baseClone(subValue, bitmask, customizer, key, value, stack)); - }); - return result; - } - - /** - * The base implementation of `_.conforms` which doesn't clone `source`. - * - * @private - * @param {Object} source The object of property predicates to conform to. - * @returns {Function} Returns the new spec function. - */ - function baseConforms(source) { - var props = keys(source); - return function(object) { - return baseConformsTo(object, source, props); - }; - } - - /** - * The base implementation of `_.conformsTo` which accepts `props` to check. - * - * @private - * @param {Object} object The object to inspect. - * @param {Object} source The object of property predicates to conform to. - * @returns {boolean} Returns `true` if `object` conforms, else `false`. - */ - function baseConformsTo(object, source, props) { - var length = props.length; - if (object == null) { - return !length; - } - object = Object(object); - while (length--) { - var key = props[length], - predicate = source[key], - value = object[key]; - - if ((value === undefined && !(key in object)) || !predicate(value)) { - return false; - } - } - return true; - } - - /** - * The base implementation of `_.delay` and `_.defer` which accepts `args` - * to provide to `func`. - * - * @private - * @param {Function} func The function to delay. - * @param {number} wait The number of milliseconds to delay invocation. - * @param {Array} args The arguments to provide to `func`. - * @returns {number|Object} Returns the timer id or timeout object. - */ - function baseDelay(func, wait, args) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - return setTimeout(function() { func.apply(undefined, args); }, wait); - } - - /** - * The base implementation of methods like `_.difference` without support - * for excluding multiple arrays or iteratee shorthands. - * - * @private - * @param {Array} array The array to inspect. - * @param {Array} values The values to exclude. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of filtered values. - */ - function baseDifference(array, values, iteratee, comparator) { - var index = -1, - includes = arrayIncludes, - isCommon = true, - length = array.length, - result = [], - valuesLength = values.length; - - if (!length) { - return result; - } - if (iteratee) { - values = arrayMap(values, baseUnary(iteratee)); - } - if (comparator) { - includes = arrayIncludesWith; - isCommon = false; - } - else if (values.length >= LARGE_ARRAY_SIZE) { - includes = cacheHas; - isCommon = false; - values = new SetCache(values); - } - outer: - while (++index < length) { - var value = array[index], - computed = iteratee == null ? value : iteratee(value); - - value = (comparator || value !== 0) ? value : 0; - if (isCommon && computed === computed) { - var valuesIndex = valuesLength; - while (valuesIndex--) { - if (values[valuesIndex] === computed) { - continue outer; - } - } - result.push(value); - } - else if (!includes(values, computed, comparator)) { - result.push(value); - } - } - return result; - } - - /** - * The base implementation of `_.forEach` without support for iteratee shorthands. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. - */ - var baseEach = createBaseEach(baseForOwn); - - /** - * The base implementation of `_.forEachRight` without support for iteratee shorthands. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. - */ - var baseEachRight = createBaseEach(baseForOwnRight, true); - - /** - * The base implementation of `_.every` without support for iteratee shorthands. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if all elements pass the predicate check, - * else `false` - */ - function baseEvery(collection, predicate) { - var result = true; - baseEach(collection, function(value, index, collection) { - result = !!predicate(value, index, collection); - return result; - }); - return result; - } - - /** - * The base implementation of methods like `_.max` and `_.min` which accepts a - * `comparator` to determine the extremum value. - * - * @private - * @param {Array} array The array to iterate over. - * @param {Function} iteratee The iteratee invoked per iteration. - * @param {Function} comparator The comparator used to compare values. - * @returns {*} Returns the extremum value. - */ - function baseExtremum(array, iteratee, comparator) { - var index = -1, - length = array.length; - - while (++index < length) { - var value = array[index], - current = iteratee(value); - - if (current != null && (computed === undefined - ? (current === current && !isSymbol(current)) - : comparator(current, computed) - )) { - var computed = current, - result = value; - } - } - return result; - } - - /** - * The base implementation of `_.fill` without an iteratee call guard. - * - * @private - * @param {Array} array The array to fill. - * @param {*} value The value to fill `array` with. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns `array`. - */ - function baseFill(array, value, start, end) { - var length = array.length; - - start = toInteger(start); - if (start < 0) { - start = -start > length ? 0 : (length + start); - } - end = (end === undefined || end > length) ? length : toInteger(end); - if (end < 0) { - end += length; - } - end = start > end ? 0 : toLength(end); - while (start < end) { - array[start++] = value; - } - return array; - } - - /** - * The base implementation of `_.filter` without support for iteratee shorthands. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {Array} Returns the new filtered array. - */ - function baseFilter(collection, predicate) { - var result = []; - baseEach(collection, function(value, index, collection) { - if (predicate(value, index, collection)) { - result.push(value); - } - }); - return result; - } - - /** - * The base implementation of `_.flatten` with support for restricting flattening. - * - * @private - * @param {Array} array The array to flatten. - * @param {number} depth The maximum recursion depth. - * @param {boolean} [predicate=isFlattenable] The function invoked per iteration. - * @param {boolean} [isStrict] Restrict to values that pass `predicate` checks. - * @param {Array} [result=[]] The initial result value. - * @returns {Array} Returns the new flattened array. - */ - function baseFlatten(array, depth, predicate, isStrict, result) { - var index = -1, - length = array.length; - - predicate || (predicate = isFlattenable); - result || (result = []); - - while (++index < length) { - var value = array[index]; - if (depth > 0 && predicate(value)) { - if (depth > 1) { - // Recursively flatten arrays (susceptible to call stack limits). - baseFlatten(value, depth - 1, predicate, isStrict, result); - } else { - arrayPush(result, value); - } - } else if (!isStrict) { - result[result.length] = value; - } - } - return result; - } - - /** - * The base implementation of `baseForOwn` which iterates over `object` - * properties returned by `keysFunc` and invokes `iteratee` for each property. - * Iteratee functions may exit iteration early by explicitly returning `false`. - * - * @private - * @param {Object} object The object to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {Function} keysFunc The function to get the keys of `object`. - * @returns {Object} Returns `object`. - */ - var baseFor = createBaseFor(); - - /** - * This function is like `baseFor` except that it iterates over properties - * in the opposite order. - * - * @private - * @param {Object} object The object to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @param {Function} keysFunc The function to get the keys of `object`. - * @returns {Object} Returns `object`. - */ - var baseForRight = createBaseFor(true); - - /** - * The base implementation of `_.forOwn` without support for iteratee shorthands. - * - * @private - * @param {Object} object The object to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Object} Returns `object`. - */ - function baseForOwn(object, iteratee) { - return object && baseFor(object, iteratee, keys); - } - - /** - * The base implementation of `_.forOwnRight` without support for iteratee shorthands. - * - * @private - * @param {Object} object The object to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Object} Returns `object`. - */ - function baseForOwnRight(object, iteratee) { - return object && baseForRight(object, iteratee, keys); - } - - /** - * The base implementation of `_.functions` which creates an array of - * `object` function property names filtered from `props`. - * - * @private - * @param {Object} object The object to inspect. - * @param {Array} props The property names to filter. - * @returns {Array} Returns the function names. - */ - function baseFunctions(object, props) { - return arrayFilter(props, function(key) { - return isFunction(object[key]); - }); - } - - /** - * The base implementation of `_.get` without support for default values. - * - * @private - * @param {Object} object The object to query. - * @param {Array|string} path The path of the property to get. - * @returns {*} Returns the resolved value. - */ - function baseGet(object, path) { - path = castPath(path, object); - - var index = 0, - length = path.length; - - while (object != null && index < length) { - object = object[toKey(path[index++])]; - } - return (index && index == length) ? object : undefined; - } - - /** - * The base implementation of `getAllKeys` and `getAllKeysIn` which uses - * `keysFunc` and `symbolsFunc` to get the enumerable property names and - * symbols of `object`. - * - * @private - * @param {Object} object The object to query. - * @param {Function} keysFunc The function to get the keys of `object`. - * @param {Function} symbolsFunc The function to get the symbols of `object`. - * @returns {Array} Returns the array of property names and symbols. - */ - function baseGetAllKeys(object, keysFunc, symbolsFunc) { - var result = keysFunc(object); - return isArray(object) ? result : arrayPush(result, symbolsFunc(object)); - } - - /** - * The base implementation of `getTag` without fallbacks for buggy environments. - * - * @private - * @param {*} value The value to query. - * @returns {string} Returns the `toStringTag`. - */ - function baseGetTag(value) { - if (value == null) { - return value === undefined ? undefinedTag : nullTag; - } - return (symToStringTag && symToStringTag in Object(value)) - ? getRawTag(value) - : objectToString(value); - } - - /** - * The base implementation of `_.gt` which doesn't coerce arguments. - * - * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is greater than `other`, - * else `false`. - */ - function baseGt(value, other) { - return value > other; - } - - /** - * The base implementation of `_.has` without support for deep paths. - * - * @private - * @param {Object} [object] The object to query. - * @param {Array|string} key The key to check. - * @returns {boolean} Returns `true` if `key` exists, else `false`. - */ - function baseHas(object, key) { - return object != null && hasOwnProperty.call(object, key); - } - - /** - * The base implementation of `_.hasIn` without support for deep paths. - * - * @private - * @param {Object} [object] The object to query. - * @param {Array|string} key The key to check. - * @returns {boolean} Returns `true` if `key` exists, else `false`. - */ - function baseHasIn(object, key) { - return object != null && key in Object(object); - } - - /** - * The base implementation of `_.inRange` which doesn't coerce arguments. - * - * @private - * @param {number} number The number to check. - * @param {number} start The start of the range. - * @param {number} end The end of the range. - * @returns {boolean} Returns `true` if `number` is in the range, else `false`. - */ - function baseInRange(number, start, end) { - return number >= nativeMin(start, end) && number < nativeMax(start, end); - } - - /** - * The base implementation of methods like `_.intersection`, without support - * for iteratee shorthands, that accepts an array of arrays to inspect. - * - * @private - * @param {Array} arrays The arrays to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of shared values. - */ - function baseIntersection(arrays, iteratee, comparator) { - var includes = comparator ? arrayIncludesWith : arrayIncludes, - length = arrays[0].length, - othLength = arrays.length, - othIndex = othLength, - caches = Array(othLength), - maxLength = Infinity, - result = []; - - while (othIndex--) { - var array = arrays[othIndex]; - if (othIndex && iteratee) { - array = arrayMap(array, baseUnary(iteratee)); - } - maxLength = nativeMin(array.length, maxLength); - caches[othIndex] = !comparator && (iteratee || (length >= 120 && array.length >= 120)) - ? new SetCache(othIndex && array) - : undefined; - } - array = arrays[0]; - - var index = -1, - seen = caches[0]; - - outer: - while (++index < length && result.length < maxLength) { - var value = array[index], - computed = iteratee ? iteratee(value) : value; - - value = (comparator || value !== 0) ? value : 0; - if (!(seen - ? cacheHas(seen, computed) - : includes(result, computed, comparator) - )) { - othIndex = othLength; - while (--othIndex) { - var cache = caches[othIndex]; - if (!(cache - ? cacheHas(cache, computed) - : includes(arrays[othIndex], computed, comparator)) - ) { - continue outer; - } - } - if (seen) { - seen.push(computed); - } - result.push(value); - } - } - return result; - } - - /** - * The base implementation of `_.invert` and `_.invertBy` which inverts - * `object` with values transformed by `iteratee` and set by `setter`. - * - * @private - * @param {Object} object The object to iterate over. - * @param {Function} setter The function to set `accumulator` values. - * @param {Function} iteratee The iteratee to transform values. - * @param {Object} accumulator The initial inverted object. - * @returns {Function} Returns `accumulator`. - */ - function baseInverter(object, setter, iteratee, accumulator) { - baseForOwn(object, function(value, key, object) { - setter(accumulator, iteratee(value), key, object); - }); - return accumulator; - } - - /** - * The base implementation of `_.invoke` without support for individual - * method arguments. - * - * @private - * @param {Object} object The object to query. - * @param {Array|string} path The path of the method to invoke. - * @param {Array} args The arguments to invoke the method with. - * @returns {*} Returns the result of the invoked method. - */ - function baseInvoke(object, path, args) { - path = castPath(path, object); - object = parent(object, path); - var func = object == null ? object : object[toKey(last(path))]; - return func == null ? undefined : apply(func, object, args); - } - - /** - * The base implementation of `_.isArguments`. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an `arguments` object, - */ - function baseIsArguments(value) { - return isObjectLike(value) && baseGetTag(value) == argsTag; - } - - /** - * The base implementation of `_.isArrayBuffer` without Node.js optimizations. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array buffer, else `false`. - */ - function baseIsArrayBuffer(value) { - return isObjectLike(value) && baseGetTag(value) == arrayBufferTag; - } - - /** - * The base implementation of `_.isDate` without Node.js optimizations. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a date object, else `false`. - */ - function baseIsDate(value) { - return isObjectLike(value) && baseGetTag(value) == dateTag; - } - - /** - * The base implementation of `_.isEqual` which supports partial comparisons - * and tracks traversed objects. - * - * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @param {boolean} bitmask The bitmask flags. - * 1 - Unordered comparison - * 2 - Partial comparison - * @param {Function} [customizer] The function to customize comparisons. - * @param {Object} [stack] Tracks traversed `value` and `other` objects. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. - */ - function baseIsEqual(value, other, bitmask, customizer, stack) { - if (value === other) { - return true; - } - if (value == null || other == null || (!isObjectLike(value) && !isObjectLike(other))) { - return value !== value && other !== other; - } - return baseIsEqualDeep(value, other, bitmask, customizer, baseIsEqual, stack); - } - - /** - * A specialized version of `baseIsEqual` for arrays and objects which performs - * deep comparisons and tracks traversed objects enabling objects with circular - * references to be compared. - * - * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} [stack] Tracks traversed `object` and `other` objects. - * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. - */ - function baseIsEqualDeep(object, other, bitmask, customizer, equalFunc, stack) { - var objIsArr = isArray(object), - othIsArr = isArray(other), - objTag = objIsArr ? arrayTag : getTag(object), - othTag = othIsArr ? arrayTag : getTag(other); - - objTag = objTag == argsTag ? objectTag : objTag; - othTag = othTag == argsTag ? objectTag : othTag; - - var objIsObj = objTag == objectTag, - othIsObj = othTag == objectTag, - isSameTag = objTag == othTag; - - if (isSameTag && isBuffer(object)) { - if (!isBuffer(other)) { - return false; - } - objIsArr = true; - objIsObj = false; - } - if (isSameTag && !objIsObj) { - stack || (stack = new Stack); - return (objIsArr || isTypedArray(object)) - ? equalArrays(object, other, bitmask, customizer, equalFunc, stack) - : equalByTag(object, other, objTag, bitmask, customizer, equalFunc, stack); - } - if (!(bitmask & COMPARE_PARTIAL_FLAG)) { - var objIsWrapped = objIsObj && hasOwnProperty.call(object, '__wrapped__'), - othIsWrapped = othIsObj && hasOwnProperty.call(other, '__wrapped__'); - - if (objIsWrapped || othIsWrapped) { - var objUnwrapped = objIsWrapped ? object.value() : object, - othUnwrapped = othIsWrapped ? other.value() : other; - - stack || (stack = new Stack); - return equalFunc(objUnwrapped, othUnwrapped, bitmask, customizer, stack); - } - } - if (!isSameTag) { - return false; - } - stack || (stack = new Stack); - return equalObjects(object, other, bitmask, customizer, equalFunc, stack); - } - - /** - * The base implementation of `_.isMap` without Node.js optimizations. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a map, else `false`. - */ - function baseIsMap(value) { - return isObjectLike(value) && getTag(value) == mapTag; - } - - /** - * The base implementation of `_.isMatch` without support for iteratee shorthands. - * - * @private - * @param {Object} object The object to inspect. - * @param {Object} source The object of property values to match. - * @param {Array} matchData The property names, values, and compare flags to match. - * @param {Function} [customizer] The function to customize comparisons. - * @returns {boolean} Returns `true` if `object` is a match, else `false`. - */ - function baseIsMatch(object, source, matchData, customizer) { - var index = matchData.length, - length = index, - noCustomizer = !customizer; - - if (object == null) { - return !length; - } - object = Object(object); - while (index--) { - var data = matchData[index]; - if ((noCustomizer && data[2]) - ? data[1] !== object[data[0]] - : !(data[0] in object) - ) { - return false; - } - } - while (++index < length) { - data = matchData[index]; - var key = data[0], - objValue = object[key], - srcValue = data[1]; - - if (noCustomizer && data[2]) { - if (objValue === undefined && !(key in object)) { - return false; - } - } else { - var stack = new Stack; - if (customizer) { - var result = customizer(objValue, srcValue, key, object, source, stack); - } - if (!(result === undefined - ? baseIsEqual(srcValue, objValue, COMPARE_PARTIAL_FLAG | COMPARE_UNORDERED_FLAG, customizer, stack) - : result - )) { - return false; - } - } - } - return true; - } - - /** - * The base implementation of `_.isNative` without bad shim checks. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a native function, - * else `false`. - */ - function baseIsNative(value) { - if (!isObject(value) || isMasked(value)) { - return false; - } - var pattern = isFunction(value) ? reIsNative : reIsHostCtor; - return pattern.test(toSource(value)); - } - - /** - * The base implementation of `_.isRegExp` without Node.js optimizations. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a regexp, else `false`. - */ - function baseIsRegExp(value) { - return isObjectLike(value) && baseGetTag(value) == regexpTag; - } - - /** - * The base implementation of `_.isSet` without Node.js optimizations. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a set, else `false`. - */ - function baseIsSet(value) { - return isObjectLike(value) && getTag(value) == setTag; - } - - /** - * The base implementation of `_.isTypedArray` without Node.js optimizations. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a typed array, else `false`. - */ - function baseIsTypedArray(value) { - return isObjectLike(value) && - isLength(value.length) && !!typedArrayTags[baseGetTag(value)]; - } - - /** - * The base implementation of `_.iteratee`. - * - * @private - * @param {*} [value=_.identity] The value to convert to an iteratee. - * @returns {Function} Returns the iteratee. - */ - function baseIteratee(value) { - // Don't store the `typeof` result in a variable to avoid a JIT bug in Safari 9. - // See https://bugs.webkit.org/show_bug.cgi?id=156034 for more details. - if (typeof value == 'function') { - return value; - } - if (value == null) { - return identity; - } - if (typeof value == 'object') { - return isArray(value) - ? baseMatchesProperty(value[0], value[1]) - : baseMatches(value); - } - return property(value); - } - - /** - * The base implementation of `_.keys` which doesn't treat sparse arrays as dense. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. - */ - function baseKeys(object) { - if (!isPrototype(object)) { - return nativeKeys(object); - } - var result = []; - for (var key in Object(object)) { - if (hasOwnProperty.call(object, key) && key != 'constructor') { - result.push(key); - } - } - return result; - } - - /** - * The base implementation of `_.keysIn` which doesn't treat sparse arrays as dense. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. - */ - function baseKeysIn(object) { - if (!isObject(object)) { - return nativeKeysIn(object); - } - var isProto = isPrototype(object), - result = []; - - for (var key in object) { - if (!(key == 'constructor' && (isProto || !hasOwnProperty.call(object, key)))) { - result.push(key); - } - } - return result; - } - - /** - * The base implementation of `_.lt` which doesn't coerce arguments. - * - * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is less than `other`, - * else `false`. - */ - function baseLt(value, other) { - return value < other; - } - - /** - * The base implementation of `_.map` without support for iteratee shorthands. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} iteratee The function invoked per iteration. - * @returns {Array} Returns the new mapped array. - */ - function baseMap(collection, iteratee) { - var index = -1, - result = isArrayLike(collection) ? Array(collection.length) : []; - - baseEach(collection, function(value, key, collection) { - result[++index] = iteratee(value, key, collection); - }); - return result; - } - - /** - * The base implementation of `_.matches` which doesn't clone `source`. - * - * @private - * @param {Object} source The object of property values to match. - * @returns {Function} Returns the new spec function. - */ - function baseMatches(source) { - var matchData = getMatchData(source); - if (matchData.length == 1 && matchData[0][2]) { - return matchesStrictComparable(matchData[0][0], matchData[0][1]); - } - return function(object) { - return object === source || baseIsMatch(object, source, matchData); - }; - } - - /** - * The base implementation of `_.matchesProperty` which doesn't clone `srcValue`. - * - * @private - * @param {string} path The path of the property to get. - * @param {*} srcValue The value to match. - * @returns {Function} Returns the new spec function. - */ - function baseMatchesProperty(path, srcValue) { - if (isKey(path) && isStrictComparable(srcValue)) { - return matchesStrictComparable(toKey(path), srcValue); - } - return function(object) { - var objValue = get(object, path); - return (objValue === undefined && objValue === srcValue) - ? hasIn(object, path) - : baseIsEqual(srcValue, objValue, COMPARE_PARTIAL_FLAG | COMPARE_UNORDERED_FLAG); - }; - } - - /** - * The base implementation of `_.merge` without support for multiple sources. - * - * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @param {number} srcIndex The index of `source`. - * @param {Function} [customizer] The function to customize merged values. - * @param {Object} [stack] Tracks traversed source values and their merged - * counterparts. - */ - function baseMerge(object, source, srcIndex, customizer, stack) { - if (object === source) { - return; - } - baseFor(source, function(srcValue, key) { - stack || (stack = new Stack); - if (isObject(srcValue)) { - baseMergeDeep(object, source, key, srcIndex, baseMerge, customizer, stack); - } - else { - var newValue = customizer - ? customizer(safeGet(object, key), srcValue, (key + ''), object, source, stack) - : undefined; - - if (newValue === undefined) { - newValue = srcValue; - } - assignMergeValue(object, key, newValue); - } - }, keysIn); - } - - /** - * A specialized version of `baseMerge` for arrays and objects which performs - * deep merges and tracks traversed objects enabling objects with circular - * references to be merged. - * - * @private - * @param {Object} object The destination object. - * @param {Object} source The source object. - * @param {string} key The key of the value to merge. - * @param {number} srcIndex The index of `source`. - * @param {Function} mergeFunc The function to merge values. - * @param {Function} [customizer] The function to customize assigned values. - * @param {Object} [stack] Tracks traversed source values and their merged - * counterparts. - */ - function baseMergeDeep(object, source, key, srcIndex, mergeFunc, customizer, stack) { - var objValue = safeGet(object, key), - srcValue = safeGet(source, key), - stacked = stack.get(srcValue); - - if (stacked) { - assignMergeValue(object, key, stacked); - return; - } - var newValue = customizer - ? customizer(objValue, srcValue, (key + ''), object, source, stack) - : undefined; - - var isCommon = newValue === undefined; - - if (isCommon) { - var isArr = isArray(srcValue), - isBuff = !isArr && isBuffer(srcValue), - isTyped = !isArr && !isBuff && isTypedArray(srcValue); - - newValue = srcValue; - if (isArr || isBuff || isTyped) { - if (isArray(objValue)) { - newValue = objValue; - } - else if (isArrayLikeObject(objValue)) { - newValue = copyArray(objValue); - } - else if (isBuff) { - isCommon = false; - newValue = cloneBuffer(srcValue, true); - } - else if (isTyped) { - isCommon = false; - newValue = cloneTypedArray(srcValue, true); - } - else { - newValue = []; - } - } - else if (isPlainObject(srcValue) || isArguments(srcValue)) { - newValue = objValue; - if (isArguments(objValue)) { - newValue = toPlainObject(objValue); - } - else if (!isObject(objValue) || isFunction(objValue)) { - newValue = initCloneObject(srcValue); - } - } - else { - isCommon = false; - } - } - if (isCommon) { - // Recursively merge objects and arrays (susceptible to call stack limits). - stack.set(srcValue, newValue); - mergeFunc(newValue, srcValue, srcIndex, customizer, stack); - stack['delete'](srcValue); - } - assignMergeValue(object, key, newValue); - } - - /** - * The base implementation of `_.nth` which doesn't coerce arguments. - * - * @private - * @param {Array} array The array to query. - * @param {number} n The index of the element to return. - * @returns {*} Returns the nth element of `array`. - */ - function baseNth(array, n) { - var length = array.length; - if (!length) { - return; - } - n += n < 0 ? length : 0; - return isIndex(n, length) ? array[n] : undefined; - } - - /** - * The base implementation of `_.orderBy` without param guards. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function[]|Object[]|string[]} iteratees The iteratees to sort by. - * @param {string[]} orders The sort orders of `iteratees`. - * @returns {Array} Returns the new sorted array. - */ - function baseOrderBy(collection, iteratees, orders) { - var index = -1; - iteratees = arrayMap(iteratees.length ? iteratees : [identity], baseUnary(getIteratee())); - - var result = baseMap(collection, function(value, key, collection) { - var criteria = arrayMap(iteratees, function(iteratee) { - return iteratee(value); - }); - return { 'criteria': criteria, 'index': ++index, 'value': value }; - }); - - return baseSortBy(result, function(object, other) { - return compareMultiple(object, other, orders); - }); - } - - /** - * The base implementation of `_.pick` without support for individual - * property identifiers. - * - * @private - * @param {Object} object The source object. - * @param {string[]} paths The property paths to pick. - * @returns {Object} Returns the new object. - */ - function basePick(object, paths) { - return basePickBy(object, paths, function(value, path) { - return hasIn(object, path); - }); - } - - /** - * The base implementation of `_.pickBy` without support for iteratee shorthands. - * - * @private - * @param {Object} object The source object. - * @param {string[]} paths The property paths to pick. - * @param {Function} predicate The function invoked per property. - * @returns {Object} Returns the new object. - */ - function basePickBy(object, paths, predicate) { - var index = -1, - length = paths.length, - result = {}; - - while (++index < length) { - var path = paths[index], - value = baseGet(object, path); - - if (predicate(value, path)) { - baseSet(result, castPath(path, object), value); - } - } - return result; - } - - /** - * A specialized version of `baseProperty` which supports deep paths. - * - * @private - * @param {Array|string} path The path of the property to get. - * @returns {Function} Returns the new accessor function. - */ - function basePropertyDeep(path) { - return function(object) { - return baseGet(object, path); - }; - } - - /** - * The base implementation of `_.pullAllBy` without support for iteratee - * shorthands. - * - * @private - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns `array`. - */ - function basePullAll(array, values, iteratee, comparator) { - var indexOf = comparator ? baseIndexOfWith : baseIndexOf, - index = -1, - length = values.length, - seen = array; - - if (array === values) { - values = copyArray(values); - } - if (iteratee) { - seen = arrayMap(array, baseUnary(iteratee)); - } - while (++index < length) { - var fromIndex = 0, - value = values[index], - computed = iteratee ? iteratee(value) : value; - - while ((fromIndex = indexOf(seen, computed, fromIndex, comparator)) > -1) { - if (seen !== array) { - splice.call(seen, fromIndex, 1); - } - splice.call(array, fromIndex, 1); - } - } - return array; - } - - /** - * The base implementation of `_.pullAt` without support for individual - * indexes or capturing the removed elements. - * - * @private - * @param {Array} array The array to modify. - * @param {number[]} indexes The indexes of elements to remove. - * @returns {Array} Returns `array`. - */ - function basePullAt(array, indexes) { - var length = array ? indexes.length : 0, - lastIndex = length - 1; - - while (length--) { - var index = indexes[length]; - if (length == lastIndex || index !== previous) { - var previous = index; - if (isIndex(index)) { - splice.call(array, index, 1); - } else { - baseUnset(array, index); - } - } - } - return array; - } - - /** - * The base implementation of `_.random` without support for returning - * floating-point numbers. - * - * @private - * @param {number} lower The lower bound. - * @param {number} upper The upper bound. - * @returns {number} Returns the random number. - */ - function baseRandom(lower, upper) { - return lower + nativeFloor(nativeRandom() * (upper - lower + 1)); - } - - /** - * The base implementation of `_.range` and `_.rangeRight` which doesn't - * coerce arguments. - * - * @private - * @param {number} start The start of the range. - * @param {number} end The end of the range. - * @param {number} step The value to increment or decrement by. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Array} Returns the range of numbers. - */ - function baseRange(start, end, step, fromRight) { - var index = -1, - length = nativeMax(nativeCeil((end - start) / (step || 1)), 0), - result = Array(length); - - while (length--) { - result[fromRight ? length : ++index] = start; - start += step; - } - return result; - } - - /** - * The base implementation of `_.repeat` which doesn't coerce arguments. - * - * @private - * @param {string} string The string to repeat. - * @param {number} n The number of times to repeat the string. - * @returns {string} Returns the repeated string. - */ - function baseRepeat(string, n) { - var result = ''; - if (!string || n < 1 || n > MAX_SAFE_INTEGER) { - return result; - } - // Leverage the exponentiation by squaring algorithm for a faster repeat. - // See https://en.wikipedia.org/wiki/Exponentiation_by_squaring for more details. - do { - if (n % 2) { - result += string; - } - n = nativeFloor(n / 2); - if (n) { - string += string; - } - } while (n); - - return result; - } - - /** - * The base implementation of `_.rest` which doesn't validate or coerce arguments. - * - * @private - * @param {Function} func The function to apply a rest parameter to. - * @param {number} [start=func.length-1] The start position of the rest parameter. - * @returns {Function} Returns the new function. - */ - function baseRest(func, start) { - return setToString(overRest(func, start, identity), func + ''); - } - - /** - * The base implementation of `_.sample`. - * - * @private - * @param {Array|Object} collection The collection to sample. - * @returns {*} Returns the random element. - */ - function baseSample(collection) { - return arraySample(values(collection)); - } - - /** - * The base implementation of `_.sampleSize` without param guards. - * - * @private - * @param {Array|Object} collection The collection to sample. - * @param {number} n The number of elements to sample. - * @returns {Array} Returns the random elements. - */ - function baseSampleSize(collection, n) { - var array = values(collection); - return shuffleSelf(array, baseClamp(n, 0, array.length)); - } - - /** - * The base implementation of `_.set`. - * - * @private - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to set. - * @param {*} value The value to set. - * @param {Function} [customizer] The function to customize path creation. - * @returns {Object} Returns `object`. - */ - function baseSet(object, path, value, customizer) { - if (!isObject(object)) { - return object; - } - path = castPath(path, object); - - var index = -1, - length = path.length, - lastIndex = length - 1, - nested = object; - - while (nested != null && ++index < length) { - var key = toKey(path[index]), - newValue = value; - - if (index != lastIndex) { - var objValue = nested[key]; - newValue = customizer ? customizer(objValue, key, nested) : undefined; - if (newValue === undefined) { - newValue = isObject(objValue) - ? objValue - : (isIndex(path[index + 1]) ? [] : {}); - } - } - assignValue(nested, key, newValue); - nested = nested[key]; - } - return object; - } - - /** - * The base implementation of `setData` without support for hot loop shorting. - * - * @private - * @param {Function} func The function to associate metadata with. - * @param {*} data The metadata. - * @returns {Function} Returns `func`. - */ - var baseSetData = !metaMap ? identity : function(func, data) { - metaMap.set(func, data); - return func; - }; - - /** - * The base implementation of `setToString` without support for hot loop shorting. - * - * @private - * @param {Function} func The function to modify. - * @param {Function} string The `toString` result. - * @returns {Function} Returns `func`. - */ - var baseSetToString = !defineProperty ? identity : function(func, string) { - return defineProperty(func, 'toString', { - 'configurable': true, - 'enumerable': false, - 'value': constant(string), - 'writable': true - }); - }; - - /** - * The base implementation of `_.shuffle`. - * - * @private - * @param {Array|Object} collection The collection to shuffle. - * @returns {Array} Returns the new shuffled array. - */ - function baseShuffle(collection) { - return shuffleSelf(values(collection)); - } - - /** - * The base implementation of `_.slice` without an iteratee call guard. - * - * @private - * @param {Array} array The array to slice. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns the slice of `array`. - */ - function baseSlice(array, start, end) { - var index = -1, - length = array.length; - - if (start < 0) { - start = -start > length ? 0 : (length + start); - } - end = end > length ? length : end; - if (end < 0) { - end += length; - } - length = start > end ? 0 : ((end - start) >>> 0); - start >>>= 0; - - var result = Array(length); - while (++index < length) { - result[index] = array[index + start]; - } - return result; - } - - /** - * The base implementation of `_.some` without support for iteratee shorthands. - * - * @private - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} predicate The function invoked per iteration. - * @returns {boolean} Returns `true` if any element passes the predicate check, - * else `false`. - */ - function baseSome(collection, predicate) { - var result; - - baseEach(collection, function(value, index, collection) { - result = predicate(value, index, collection); - return !result; - }); - return !!result; - } - - /** - * The base implementation of `_.sortedIndex` and `_.sortedLastIndex` which - * performs a binary search of `array` to determine the index at which `value` - * should be inserted into `array` in order to maintain its sort order. - * - * @private - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {boolean} [retHighest] Specify returning the highest qualified index. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. - */ - function baseSortedIndex(array, value, retHighest) { - var low = 0, - high = array == null ? low : array.length; - - if (typeof value == 'number' && value === value && high <= HALF_MAX_ARRAY_LENGTH) { - while (low < high) { - var mid = (low + high) >>> 1, - computed = array[mid]; - - if (computed !== null && !isSymbol(computed) && - (retHighest ? (computed <= value) : (computed < value))) { - low = mid + 1; - } else { - high = mid; - } - } - return high; - } - return baseSortedIndexBy(array, value, identity, retHighest); - } - - /** - * The base implementation of `_.sortedIndexBy` and `_.sortedLastIndexBy` - * which invokes `iteratee` for `value` and each element of `array` to compute - * their sort ranking. The iteratee is invoked with one argument; (value). - * - * @private - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {Function} iteratee The iteratee invoked per element. - * @param {boolean} [retHighest] Specify returning the highest qualified index. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. - */ - function baseSortedIndexBy(array, value, iteratee, retHighest) { - value = iteratee(value); - - var low = 0, - high = array == null ? 0 : array.length, - valIsNaN = value !== value, - valIsNull = value === null, - valIsSymbol = isSymbol(value), - valIsUndefined = value === undefined; - - while (low < high) { - var mid = nativeFloor((low + high) / 2), - computed = iteratee(array[mid]), - othIsDefined = computed !== undefined, - othIsNull = computed === null, - othIsReflexive = computed === computed, - othIsSymbol = isSymbol(computed); - - if (valIsNaN) { - var setLow = retHighest || othIsReflexive; - } else if (valIsUndefined) { - setLow = othIsReflexive && (retHighest || othIsDefined); - } else if (valIsNull) { - setLow = othIsReflexive && othIsDefined && (retHighest || !othIsNull); - } else if (valIsSymbol) { - setLow = othIsReflexive && othIsDefined && !othIsNull && (retHighest || !othIsSymbol); - } else if (othIsNull || othIsSymbol) { - setLow = false; - } else { - setLow = retHighest ? (computed <= value) : (computed < value); - } - if (setLow) { - low = mid + 1; - } else { - high = mid; - } - } - return nativeMin(high, MAX_ARRAY_INDEX); - } - - /** - * The base implementation of `_.sortedUniq` and `_.sortedUniqBy` without - * support for iteratee shorthands. - * - * @private - * @param {Array} array The array to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @returns {Array} Returns the new duplicate free array. - */ - function baseSortedUniq(array, iteratee) { - var index = -1, - length = array.length, - resIndex = 0, - result = []; - - while (++index < length) { - var value = array[index], - computed = iteratee ? iteratee(value) : value; - - if (!index || !eq(computed, seen)) { - var seen = computed; - result[resIndex++] = value === 0 ? 0 : value; - } - } - return result; - } - - /** - * The base implementation of `_.toNumber` which doesn't ensure correct - * conversions of binary, hexadecimal, or octal string values. - * - * @private - * @param {*} value The value to process. - * @returns {number} Returns the number. - */ - function baseToNumber(value) { - if (typeof value == 'number') { - return value; - } - if (isSymbol(value)) { - return NAN; - } - return +value; - } - - /** - * The base implementation of `_.toString` which doesn't convert nullish - * values to empty strings. - * - * @private - * @param {*} value The value to process. - * @returns {string} Returns the string. - */ - function baseToString(value) { - // Exit early for strings to avoid a performance hit in some environments. - if (typeof value == 'string') { - return value; - } - if (isArray(value)) { - // Recursively convert values (susceptible to call stack limits). - return arrayMap(value, baseToString) + ''; - } - if (isSymbol(value)) { - return symbolToString ? symbolToString.call(value) : ''; - } - var result = (value + ''); - return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; - } - - /** - * The base implementation of `_.uniqBy` without support for iteratee shorthands. - * - * @private - * @param {Array} array The array to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new duplicate free array. - */ - function baseUniq(array, iteratee, comparator) { - var index = -1, - includes = arrayIncludes, - length = array.length, - isCommon = true, - result = [], - seen = result; - - if (comparator) { - isCommon = false; - includes = arrayIncludesWith; - } - else if (length >= LARGE_ARRAY_SIZE) { - var set = iteratee ? null : createSet(array); - if (set) { - return setToArray(set); - } - isCommon = false; - includes = cacheHas; - seen = new SetCache; - } - else { - seen = iteratee ? [] : result; - } - outer: - while (++index < length) { - var value = array[index], - computed = iteratee ? iteratee(value) : value; - - value = (comparator || value !== 0) ? value : 0; - if (isCommon && computed === computed) { - var seenIndex = seen.length; - while (seenIndex--) { - if (seen[seenIndex] === computed) { - continue outer; - } - } - if (iteratee) { - seen.push(computed); - } - result.push(value); - } - else if (!includes(seen, computed, comparator)) { - if (seen !== result) { - seen.push(computed); - } - result.push(value); - } - } - return result; - } - - /** - * The base implementation of `_.unset`. - * - * @private - * @param {Object} object The object to modify. - * @param {Array|string} path The property path to unset. - * @returns {boolean} Returns `true` if the property is deleted, else `false`. - */ - function baseUnset(object, path) { - path = castPath(path, object); - object = parent(object, path); - return object == null || delete object[toKey(last(path))]; - } - - /** - * The base implementation of `_.update`. - * - * @private - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to update. - * @param {Function} updater The function to produce the updated value. - * @param {Function} [customizer] The function to customize path creation. - * @returns {Object} Returns `object`. - */ - function baseUpdate(object, path, updater, customizer) { - return baseSet(object, path, updater(baseGet(object, path)), customizer); - } - - /** - * The base implementation of methods like `_.dropWhile` and `_.takeWhile` - * without support for iteratee shorthands. - * - * @private - * @param {Array} array The array to query. - * @param {Function} predicate The function invoked per iteration. - * @param {boolean} [isDrop] Specify dropping elements instead of taking them. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Array} Returns the slice of `array`. - */ - function baseWhile(array, predicate, isDrop, fromRight) { - var length = array.length, - index = fromRight ? length : -1; - - while ((fromRight ? index-- : ++index < length) && - predicate(array[index], index, array)) {} - - return isDrop - ? baseSlice(array, (fromRight ? 0 : index), (fromRight ? index + 1 : length)) - : baseSlice(array, (fromRight ? index + 1 : 0), (fromRight ? length : index)); - } - - /** - * The base implementation of `wrapperValue` which returns the result of - * performing a sequence of actions on the unwrapped `value`, where each - * successive action is supplied the return value of the previous. - * - * @private - * @param {*} value The unwrapped value. - * @param {Array} actions Actions to perform to resolve the unwrapped value. - * @returns {*} Returns the resolved value. - */ - function baseWrapperValue(value, actions) { - var result = value; - if (result instanceof LazyWrapper) { - result = result.value(); - } - return arrayReduce(actions, function(result, action) { - return action.func.apply(action.thisArg, arrayPush([result], action.args)); - }, result); - } - - /** - * The base implementation of methods like `_.xor`, without support for - * iteratee shorthands, that accepts an array of arrays to inspect. - * - * @private - * @param {Array} arrays The arrays to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of values. - */ - function baseXor(arrays, iteratee, comparator) { - var length = arrays.length; - if (length < 2) { - return length ? baseUniq(arrays[0]) : []; - } - var index = -1, - result = Array(length); - - while (++index < length) { - var array = arrays[index], - othIndex = -1; - - while (++othIndex < length) { - if (othIndex != index) { - result[index] = baseDifference(result[index] || array, arrays[othIndex], iteratee, comparator); - } - } - } - return baseUniq(baseFlatten(result, 1), iteratee, comparator); - } - - /** - * This base implementation of `_.zipObject` which assigns values using `assignFunc`. - * - * @private - * @param {Array} props The property identifiers. - * @param {Array} values The property values. - * @param {Function} assignFunc The function to assign values. - * @returns {Object} Returns the new object. - */ - function baseZipObject(props, values, assignFunc) { - var index = -1, - length = props.length, - valsLength = values.length, - result = {}; - - while (++index < length) { - var value = index < valsLength ? values[index] : undefined; - assignFunc(result, props[index], value); - } - return result; - } - - /** - * Casts `value` to an empty array if it's not an array like object. - * - * @private - * @param {*} value The value to inspect. - * @returns {Array|Object} Returns the cast array-like object. - */ - function castArrayLikeObject(value) { - return isArrayLikeObject(value) ? value : []; - } - - /** - * Casts `value` to `identity` if it's not a function. - * - * @private - * @param {*} value The value to inspect. - * @returns {Function} Returns cast function. - */ - function castFunction(value) { - return typeof value == 'function' ? value : identity; - } - - /** - * Casts `value` to a path array if it's not one. - * - * @private - * @param {*} value The value to inspect. - * @param {Object} [object] The object to query keys on. - * @returns {Array} Returns the cast property path array. - */ - function castPath(value, object) { - if (isArray(value)) { - return value; - } - return isKey(value, object) ? [value] : stringToPath(toString(value)); - } - - /** - * A `baseRest` alias which can be replaced with `identity` by module - * replacement plugins. - * - * @private - * @type {Function} - * @param {Function} func The function to apply a rest parameter to. - * @returns {Function} Returns the new function. - */ - var castRest = baseRest; - - /** - * Casts `array` to a slice if it's needed. - * - * @private - * @param {Array} array The array to inspect. - * @param {number} start The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns the cast slice. - */ - function castSlice(array, start, end) { - var length = array.length; - end = end === undefined ? length : end; - return (!start && end >= length) ? array : baseSlice(array, start, end); - } - - /** - * A simple wrapper around the global [`clearTimeout`](https://mdn.io/clearTimeout). - * - * @private - * @param {number|Object} id The timer id or timeout object of the timer to clear. - */ - var clearTimeout = ctxClearTimeout || function(id) { - return root.clearTimeout(id); - }; - - /** - * Creates a clone of `buffer`. - * - * @private - * @param {Buffer} buffer The buffer to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Buffer} Returns the cloned buffer. - */ - function cloneBuffer(buffer, isDeep) { - if (isDeep) { - return buffer.slice(); - } - var length = buffer.length, - result = allocUnsafe ? allocUnsafe(length) : new buffer.constructor(length); - - buffer.copy(result); - return result; - } - - /** - * Creates a clone of `arrayBuffer`. - * - * @private - * @param {ArrayBuffer} arrayBuffer The array buffer to clone. - * @returns {ArrayBuffer} Returns the cloned array buffer. - */ - function cloneArrayBuffer(arrayBuffer) { - var result = new arrayBuffer.constructor(arrayBuffer.byteLength); - new Uint8Array(result).set(new Uint8Array(arrayBuffer)); - return result; - } - - /** - * Creates a clone of `dataView`. - * - * @private - * @param {Object} dataView The data view to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Object} Returns the cloned data view. - */ - function cloneDataView(dataView, isDeep) { - var buffer = isDeep ? cloneArrayBuffer(dataView.buffer) : dataView.buffer; - return new dataView.constructor(buffer, dataView.byteOffset, dataView.byteLength); - } - - /** - * Creates a clone of `regexp`. - * - * @private - * @param {Object} regexp The regexp to clone. - * @returns {Object} Returns the cloned regexp. - */ - function cloneRegExp(regexp) { - var result = new regexp.constructor(regexp.source, reFlags.exec(regexp)); - result.lastIndex = regexp.lastIndex; - return result; - } - - /** - * Creates a clone of the `symbol` object. - * - * @private - * @param {Object} symbol The symbol object to clone. - * @returns {Object} Returns the cloned symbol object. - */ - function cloneSymbol(symbol) { - return symbolValueOf ? Object(symbolValueOf.call(symbol)) : {}; - } - - /** - * Creates a clone of `typedArray`. - * - * @private - * @param {Object} typedArray The typed array to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Object} Returns the cloned typed array. - */ - function cloneTypedArray(typedArray, isDeep) { - var buffer = isDeep ? cloneArrayBuffer(typedArray.buffer) : typedArray.buffer; - return new typedArray.constructor(buffer, typedArray.byteOffset, typedArray.length); - } - - /** - * Compares values to sort them in ascending order. - * - * @private - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {number} Returns the sort order indicator for `value`. - */ - function compareAscending(value, other) { - if (value !== other) { - var valIsDefined = value !== undefined, - valIsNull = value === null, - valIsReflexive = value === value, - valIsSymbol = isSymbol(value); - - var othIsDefined = other !== undefined, - othIsNull = other === null, - othIsReflexive = other === other, - othIsSymbol = isSymbol(other); - - if ((!othIsNull && !othIsSymbol && !valIsSymbol && value > other) || - (valIsSymbol && othIsDefined && othIsReflexive && !othIsNull && !othIsSymbol) || - (valIsNull && othIsDefined && othIsReflexive) || - (!valIsDefined && othIsReflexive) || - !valIsReflexive) { - return 1; - } - if ((!valIsNull && !valIsSymbol && !othIsSymbol && value < other) || - (othIsSymbol && valIsDefined && valIsReflexive && !valIsNull && !valIsSymbol) || - (othIsNull && valIsDefined && valIsReflexive) || - (!othIsDefined && valIsReflexive) || - !othIsReflexive) { - return -1; - } - } - return 0; - } - - /** - * Used by `_.orderBy` to compare multiple properties of a value to another - * and stable sort them. - * - * If `orders` is unspecified, all values are sorted in ascending order. Otherwise, - * specify an order of "desc" for descending or "asc" for ascending sort order - * of corresponding values. - * - * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {boolean[]|string[]} orders The order to sort by for each property. - * @returns {number} Returns the sort order indicator for `object`. - */ - function compareMultiple(object, other, orders) { - var index = -1, - objCriteria = object.criteria, - othCriteria = other.criteria, - length = objCriteria.length, - ordersLength = orders.length; - - while (++index < length) { - var result = compareAscending(objCriteria[index], othCriteria[index]); - if (result) { - if (index >= ordersLength) { - return result; - } - var order = orders[index]; - return result * (order == 'desc' ? -1 : 1); - } - } - // Fixes an `Array#sort` bug in the JS engine embedded in Adobe applications - // that causes it, under certain circumstances, to provide the same value for - // `object` and `other`. See https://github.com/jashkenas/underscore/pull/1247 - // for more details. - // - // This also ensures a stable sort in V8 and other engines. - // See https://bugs.chromium.org/p/v8/issues/detail?id=90 for more details. - return object.index - other.index; - } - - /** - * Creates an array that is the composition of partially applied arguments, - * placeholders, and provided arguments into a single array of arguments. - * - * @private - * @param {Array} args The provided arguments. - * @param {Array} partials The arguments to prepend to those provided. - * @param {Array} holders The `partials` placeholder indexes. - * @params {boolean} [isCurried] Specify composing for a curried function. - * @returns {Array} Returns the new array of composed arguments. - */ - function composeArgs(args, partials, holders, isCurried) { - var argsIndex = -1, - argsLength = args.length, - holdersLength = holders.length, - leftIndex = -1, - leftLength = partials.length, - rangeLength = nativeMax(argsLength - holdersLength, 0), - result = Array(leftLength + rangeLength), - isUncurried = !isCurried; - - while (++leftIndex < leftLength) { - result[leftIndex] = partials[leftIndex]; - } - while (++argsIndex < holdersLength) { - if (isUncurried || argsIndex < argsLength) { - result[holders[argsIndex]] = args[argsIndex]; - } - } - while (rangeLength--) { - result[leftIndex++] = args[argsIndex++]; - } - return result; - } - - /** - * This function is like `composeArgs` except that the arguments composition - * is tailored for `_.partialRight`. - * - * @private - * @param {Array} args The provided arguments. - * @param {Array} partials The arguments to append to those provided. - * @param {Array} holders The `partials` placeholder indexes. - * @params {boolean} [isCurried] Specify composing for a curried function. - * @returns {Array} Returns the new array of composed arguments. - */ - function composeArgsRight(args, partials, holders, isCurried) { - var argsIndex = -1, - argsLength = args.length, - holdersIndex = -1, - holdersLength = holders.length, - rightIndex = -1, - rightLength = partials.length, - rangeLength = nativeMax(argsLength - holdersLength, 0), - result = Array(rangeLength + rightLength), - isUncurried = !isCurried; - - while (++argsIndex < rangeLength) { - result[argsIndex] = args[argsIndex]; - } - var offset = argsIndex; - while (++rightIndex < rightLength) { - result[offset + rightIndex] = partials[rightIndex]; - } - while (++holdersIndex < holdersLength) { - if (isUncurried || argsIndex < argsLength) { - result[offset + holders[holdersIndex]] = args[argsIndex++]; - } - } - return result; - } - - /** - * Copies the values of `source` to `array`. - * - * @private - * @param {Array} source The array to copy values from. - * @param {Array} [array=[]] The array to copy values to. - * @returns {Array} Returns `array`. - */ - function copyArray(source, array) { - var index = -1, - length = source.length; - - array || (array = Array(length)); - while (++index < length) { - array[index] = source[index]; - } - return array; - } - - /** - * Copies properties of `source` to `object`. - * - * @private - * @param {Object} source The object to copy properties from. - * @param {Array} props The property identifiers to copy. - * @param {Object} [object={}] The object to copy properties to. - * @param {Function} [customizer] The function to customize copied values. - * @returns {Object} Returns `object`. - */ - function copyObject(source, props, object, customizer) { - var isNew = !object; - object || (object = {}); - - var index = -1, - length = props.length; - - while (++index < length) { - var key = props[index]; - - var newValue = customizer - ? customizer(object[key], source[key], key, object, source) - : undefined; - - if (newValue === undefined) { - newValue = source[key]; - } - if (isNew) { - baseAssignValue(object, key, newValue); - } else { - assignValue(object, key, newValue); - } - } - return object; - } - - /** - * Copies own symbols of `source` to `object`. - * - * @private - * @param {Object} source The object to copy symbols from. - * @param {Object} [object={}] The object to copy symbols to. - * @returns {Object} Returns `object`. - */ - function copySymbols(source, object) { - return copyObject(source, getSymbols(source), object); - } - - /** - * Copies own and inherited symbols of `source` to `object`. - * - * @private - * @param {Object} source The object to copy symbols from. - * @param {Object} [object={}] The object to copy symbols to. - * @returns {Object} Returns `object`. - */ - function copySymbolsIn(source, object) { - return copyObject(source, getSymbolsIn(source), object); - } - - /** - * Creates a function like `_.groupBy`. - * - * @private - * @param {Function} setter The function to set accumulator values. - * @param {Function} [initializer] The accumulator object initializer. - * @returns {Function} Returns the new aggregator function. - */ - function createAggregator(setter, initializer) { - return function(collection, iteratee) { - var func = isArray(collection) ? arrayAggregator : baseAggregator, - accumulator = initializer ? initializer() : {}; - - return func(collection, setter, getIteratee(iteratee, 2), accumulator); - }; - } - - /** - * Creates a function like `_.assign`. - * - * @private - * @param {Function} assigner The function to assign values. - * @returns {Function} Returns the new assigner function. - */ - function createAssigner(assigner) { - return baseRest(function(object, sources) { - var index = -1, - length = sources.length, - customizer = length > 1 ? sources[length - 1] : undefined, - guard = length > 2 ? sources[2] : undefined; - - customizer = (assigner.length > 3 && typeof customizer == 'function') - ? (length--, customizer) - : undefined; - - if (guard && isIterateeCall(sources[0], sources[1], guard)) { - customizer = length < 3 ? undefined : customizer; - length = 1; - } - object = Object(object); - while (++index < length) { - var source = sources[index]; - if (source) { - assigner(object, source, index, customizer); - } - } - return object; - }); - } - - /** - * Creates a `baseEach` or `baseEachRight` function. - * - * @private - * @param {Function} eachFunc The function to iterate over a collection. - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new base function. - */ - function createBaseEach(eachFunc, fromRight) { - return function(collection, iteratee) { - if (collection == null) { - return collection; - } - if (!isArrayLike(collection)) { - return eachFunc(collection, iteratee); - } - var length = collection.length, - index = fromRight ? length : -1, - iterable = Object(collection); - - while ((fromRight ? index-- : ++index < length)) { - if (iteratee(iterable[index], index, iterable) === false) { - break; - } - } - return collection; - }; - } - - /** - * Creates a base function for methods like `_.forIn` and `_.forOwn`. - * - * @private - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new base function. - */ - function createBaseFor(fromRight) { - return function(object, iteratee, keysFunc) { - var index = -1, - iterable = Object(object), - props = keysFunc(object), - length = props.length; - - while (length--) { - var key = props[fromRight ? length : ++index]; - if (iteratee(iterable[key], key, iterable) === false) { - break; - } - } - return object; - }; - } - - /** - * Creates a function that wraps `func` to invoke it with the optional `this` - * binding of `thisArg`. - * - * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {*} [thisArg] The `this` binding of `func`. - * @returns {Function} Returns the new wrapped function. - */ - function createBind(func, bitmask, thisArg) { - var isBind = bitmask & WRAP_BIND_FLAG, - Ctor = createCtor(func); - - function wrapper() { - var fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; - return fn.apply(isBind ? thisArg : this, arguments); - } - return wrapper; - } - - /** - * Creates a function like `_.lowerFirst`. - * - * @private - * @param {string} methodName The name of the `String` case method to use. - * @returns {Function} Returns the new case function. - */ - function createCaseFirst(methodName) { - return function(string) { - string = toString(string); - - var strSymbols = hasUnicode(string) - ? stringToArray(string) - : undefined; - - var chr = strSymbols - ? strSymbols[0] - : string.charAt(0); - - var trailing = strSymbols - ? castSlice(strSymbols, 1).join('') - : string.slice(1); - - return chr[methodName]() + trailing; - }; - } - - /** - * Creates a function like `_.camelCase`. - * - * @private - * @param {Function} callback The function to combine each word. - * @returns {Function} Returns the new compounder function. - */ - function createCompounder(callback) { - return function(string) { - return arrayReduce(words(deburr(string).replace(reApos, '')), callback, ''); - }; - } - - /** - * Creates a function that produces an instance of `Ctor` regardless of - * whether it was invoked as part of a `new` expression or by `call` or `apply`. - * - * @private - * @param {Function} Ctor The constructor to wrap. - * @returns {Function} Returns the new wrapped function. - */ - function createCtor(Ctor) { - return function() { - // Use a `switch` statement to work with class constructors. See - // http://ecma-international.org/ecma-262/7.0/#sec-ecmascript-function-objects-call-thisargument-argumentslist - // for more details. - var args = arguments; - switch (args.length) { - case 0: return new Ctor; - case 1: return new Ctor(args[0]); - case 2: return new Ctor(args[0], args[1]); - case 3: return new Ctor(args[0], args[1], args[2]); - case 4: return new Ctor(args[0], args[1], args[2], args[3]); - case 5: return new Ctor(args[0], args[1], args[2], args[3], args[4]); - case 6: return new Ctor(args[0], args[1], args[2], args[3], args[4], args[5]); - case 7: return new Ctor(args[0], args[1], args[2], args[3], args[4], args[5], args[6]); - } - var thisBinding = baseCreate(Ctor.prototype), - result = Ctor.apply(thisBinding, args); - - // Mimic the constructor's `return` behavior. - // See https://es5.github.io/#x13.2.2 for more details. - return isObject(result) ? result : thisBinding; - }; - } - - /** - * Creates a function that wraps `func` to enable currying. - * - * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {number} arity The arity of `func`. - * @returns {Function} Returns the new wrapped function. - */ - function createCurry(func, bitmask, arity) { - var Ctor = createCtor(func); - - function wrapper() { - var length = arguments.length, - args = Array(length), - index = length, - placeholder = getHolder(wrapper); - - while (index--) { - args[index] = arguments[index]; - } - var holders = (length < 3 && args[0] !== placeholder && args[length - 1] !== placeholder) - ? [] - : replaceHolders(args, placeholder); - - length -= holders.length; - if (length < arity) { - return createRecurry( - func, bitmask, createHybrid, wrapper.placeholder, undefined, - args, holders, undefined, undefined, arity - length); - } - var fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; - return apply(fn, this, args); - } - return wrapper; - } - - /** - * Creates a `_.find` or `_.findLast` function. - * - * @private - * @param {Function} findIndexFunc The function to find the collection index. - * @returns {Function} Returns the new find function. - */ - function createFind(findIndexFunc) { - return function(collection, predicate, fromIndex) { - var iterable = Object(collection); - if (!isArrayLike(collection)) { - var iteratee = getIteratee(predicate, 3); - collection = keys(collection); - predicate = function(key) { return iteratee(iterable[key], key, iterable); }; - } - var index = findIndexFunc(collection, predicate, fromIndex); - return index > -1 ? iterable[iteratee ? collection[index] : index] : undefined; - }; - } - - /** - * Creates a `_.flow` or `_.flowRight` function. - * - * @private - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new flow function. - */ - function createFlow(fromRight) { - return flatRest(function(funcs) { - var length = funcs.length, - index = length, - prereq = LodashWrapper.prototype.thru; - - if (fromRight) { - funcs.reverse(); - } - while (index--) { - var func = funcs[index]; - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - if (prereq && !wrapper && getFuncName(func) == 'wrapper') { - var wrapper = new LodashWrapper([], true); - } - } - index = wrapper ? index : length; - while (++index < length) { - func = funcs[index]; - - var funcName = getFuncName(func), - data = funcName == 'wrapper' ? getData(func) : undefined; - - if (data && isLaziable(data[0]) && - data[1] == (WRAP_ARY_FLAG | WRAP_CURRY_FLAG | WRAP_PARTIAL_FLAG | WRAP_REARG_FLAG) && - !data[4].length && data[9] == 1 - ) { - wrapper = wrapper[getFuncName(data[0])].apply(wrapper, data[3]); - } else { - wrapper = (func.length == 1 && isLaziable(func)) - ? wrapper[funcName]() - : wrapper.thru(func); - } - } - return function() { - var args = arguments, - value = args[0]; - - if (wrapper && args.length == 1 && isArray(value)) { - return wrapper.plant(value).value(); - } - var index = 0, - result = length ? funcs[index].apply(this, args) : value; - - while (++index < length) { - result = funcs[index].call(this, result); - } - return result; - }; - }); - } - - /** - * Creates a function that wraps `func` to invoke it with optional `this` - * binding of `thisArg`, partial application, and currying. - * - * @private - * @param {Function|string} func The function or method name to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {*} [thisArg] The `this` binding of `func`. - * @param {Array} [partials] The arguments to prepend to those provided to - * the new function. - * @param {Array} [holders] The `partials` placeholder indexes. - * @param {Array} [partialsRight] The arguments to append to those provided - * to the new function. - * @param {Array} [holdersRight] The `partialsRight` placeholder indexes. - * @param {Array} [argPos] The argument positions of the new function. - * @param {number} [ary] The arity cap of `func`. - * @param {number} [arity] The arity of `func`. - * @returns {Function} Returns the new wrapped function. - */ - function createHybrid(func, bitmask, thisArg, partials, holders, partialsRight, holdersRight, argPos, ary, arity) { - var isAry = bitmask & WRAP_ARY_FLAG, - isBind = bitmask & WRAP_BIND_FLAG, - isBindKey = bitmask & WRAP_BIND_KEY_FLAG, - isCurried = bitmask & (WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG), - isFlip = bitmask & WRAP_FLIP_FLAG, - Ctor = isBindKey ? undefined : createCtor(func); - - function wrapper() { - var length = arguments.length, - args = Array(length), - index = length; - - while (index--) { - args[index] = arguments[index]; - } - if (isCurried) { - var placeholder = getHolder(wrapper), - holdersCount = countHolders(args, placeholder); - } - if (partials) { - args = composeArgs(args, partials, holders, isCurried); - } - if (partialsRight) { - args = composeArgsRight(args, partialsRight, holdersRight, isCurried); - } - length -= holdersCount; - if (isCurried && length < arity) { - var newHolders = replaceHolders(args, placeholder); - return createRecurry( - func, bitmask, createHybrid, wrapper.placeholder, thisArg, - args, newHolders, argPos, ary, arity - length - ); - } - var thisBinding = isBind ? thisArg : this, - fn = isBindKey ? thisBinding[func] : func; - - length = args.length; - if (argPos) { - args = reorder(args, argPos); - } else if (isFlip && length > 1) { - args.reverse(); - } - if (isAry && ary < length) { - args.length = ary; - } - if (this && this !== root && this instanceof wrapper) { - fn = Ctor || createCtor(fn); - } - return fn.apply(thisBinding, args); - } - return wrapper; - } - - /** - * Creates a function like `_.invertBy`. - * - * @private - * @param {Function} setter The function to set accumulator values. - * @param {Function} toIteratee The function to resolve iteratees. - * @returns {Function} Returns the new inverter function. - */ - function createInverter(setter, toIteratee) { - return function(object, iteratee) { - return baseInverter(object, setter, toIteratee(iteratee), {}); - }; - } - - /** - * Creates a function that performs a mathematical operation on two values. - * - * @private - * @param {Function} operator The function to perform the operation. - * @param {number} [defaultValue] The value used for `undefined` arguments. - * @returns {Function} Returns the new mathematical operation function. - */ - function createMathOperation(operator, defaultValue) { - return function(value, other) { - var result; - if (value === undefined && other === undefined) { - return defaultValue; - } - if (value !== undefined) { - result = value; - } - if (other !== undefined) { - if (result === undefined) { - return other; - } - if (typeof value == 'string' || typeof other == 'string') { - value = baseToString(value); - other = baseToString(other); - } else { - value = baseToNumber(value); - other = baseToNumber(other); - } - result = operator(value, other); - } - return result; - }; - } - - /** - * Creates a function like `_.over`. - * - * @private - * @param {Function} arrayFunc The function to iterate over iteratees. - * @returns {Function} Returns the new over function. - */ - function createOver(arrayFunc) { - return flatRest(function(iteratees) { - iteratees = arrayMap(iteratees, baseUnary(getIteratee())); - return baseRest(function(args) { - var thisArg = this; - return arrayFunc(iteratees, function(iteratee) { - return apply(iteratee, thisArg, args); - }); - }); - }); - } - - /** - * Creates the padding for `string` based on `length`. The `chars` string - * is truncated if the number of characters exceeds `length`. - * - * @private - * @param {number} length The padding length. - * @param {string} [chars=' '] The string used as padding. - * @returns {string} Returns the padding for `string`. - */ - function createPadding(length, chars) { - chars = chars === undefined ? ' ' : baseToString(chars); - - var charsLength = chars.length; - if (charsLength < 2) { - return charsLength ? baseRepeat(chars, length) : chars; - } - var result = baseRepeat(chars, nativeCeil(length / stringSize(chars))); - return hasUnicode(chars) - ? castSlice(stringToArray(result), 0, length).join('') - : result.slice(0, length); - } - - /** - * Creates a function that wraps `func` to invoke it with the `this` binding - * of `thisArg` and `partials` prepended to the arguments it receives. - * - * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {*} thisArg The `this` binding of `func`. - * @param {Array} partials The arguments to prepend to those provided to - * the new function. - * @returns {Function} Returns the new wrapped function. - */ - function createPartial(func, bitmask, thisArg, partials) { - var isBind = bitmask & WRAP_BIND_FLAG, - Ctor = createCtor(func); - - function wrapper() { - var argsIndex = -1, - argsLength = arguments.length, - leftIndex = -1, - leftLength = partials.length, - args = Array(leftLength + argsLength), - fn = (this && this !== root && this instanceof wrapper) ? Ctor : func; - - while (++leftIndex < leftLength) { - args[leftIndex] = partials[leftIndex]; - } - while (argsLength--) { - args[leftIndex++] = arguments[++argsIndex]; - } - return apply(fn, isBind ? thisArg : this, args); - } - return wrapper; - } - - /** - * Creates a `_.range` or `_.rangeRight` function. - * - * @private - * @param {boolean} [fromRight] Specify iterating from right to left. - * @returns {Function} Returns the new range function. - */ - function createRange(fromRight) { - return function(start, end, step) { - if (step && typeof step != 'number' && isIterateeCall(start, end, step)) { - end = step = undefined; - } - // Ensure the sign of `-0` is preserved. - start = toFinite(start); - if (end === undefined) { - end = start; - start = 0; - } else { - end = toFinite(end); - } - step = step === undefined ? (start < end ? 1 : -1) : toFinite(step); - return baseRange(start, end, step, fromRight); - }; - } - - /** - * Creates a function that performs a relational operation on two values. - * - * @private - * @param {Function} operator The function to perform the operation. - * @returns {Function} Returns the new relational operation function. - */ - function createRelationalOperation(operator) { - return function(value, other) { - if (!(typeof value == 'string' && typeof other == 'string')) { - value = toNumber(value); - other = toNumber(other); - } - return operator(value, other); - }; - } - - /** - * Creates a function that wraps `func` to continue currying. - * - * @private - * @param {Function} func The function to wrap. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @param {Function} wrapFunc The function to create the `func` wrapper. - * @param {*} placeholder The placeholder value. - * @param {*} [thisArg] The `this` binding of `func`. - * @param {Array} [partials] The arguments to prepend to those provided to - * the new function. - * @param {Array} [holders] The `partials` placeholder indexes. - * @param {Array} [argPos] The argument positions of the new function. - * @param {number} [ary] The arity cap of `func`. - * @param {number} [arity] The arity of `func`. - * @returns {Function} Returns the new wrapped function. - */ - function createRecurry(func, bitmask, wrapFunc, placeholder, thisArg, partials, holders, argPos, ary, arity) { - var isCurry = bitmask & WRAP_CURRY_FLAG, - newHolders = isCurry ? holders : undefined, - newHoldersRight = isCurry ? undefined : holders, - newPartials = isCurry ? partials : undefined, - newPartialsRight = isCurry ? undefined : partials; - - bitmask |= (isCurry ? WRAP_PARTIAL_FLAG : WRAP_PARTIAL_RIGHT_FLAG); - bitmask &= ~(isCurry ? WRAP_PARTIAL_RIGHT_FLAG : WRAP_PARTIAL_FLAG); - - if (!(bitmask & WRAP_CURRY_BOUND_FLAG)) { - bitmask &= ~(WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG); - } - var newData = [ - func, bitmask, thisArg, newPartials, newHolders, newPartialsRight, - newHoldersRight, argPos, ary, arity - ]; - - var result = wrapFunc.apply(undefined, newData); - if (isLaziable(func)) { - setData(result, newData); - } - result.placeholder = placeholder; - return setWrapToString(result, func, bitmask); - } - - /** - * Creates a function like `_.round`. - * - * @private - * @param {string} methodName The name of the `Math` method to use when rounding. - * @returns {Function} Returns the new round function. - */ - function createRound(methodName) { - var func = Math[methodName]; - return function(number, precision) { - number = toNumber(number); - precision = precision == null ? 0 : nativeMin(toInteger(precision), 292); - if (precision && nativeIsFinite(number)) { - // Shift with exponential notation to avoid floating-point issues. - // See [MDN](https://mdn.io/round#Examples) for more details. - var pair = (toString(number) + 'e').split('e'), - value = func(pair[0] + 'e' + (+pair[1] + precision)); - - pair = (toString(value) + 'e').split('e'); - return +(pair[0] + 'e' + (+pair[1] - precision)); - } - return func(number); - }; - } - - /** - * Creates a set object of `values`. - * - * @private - * @param {Array} values The values to add to the set. - * @returns {Object} Returns the new set. - */ - var createSet = !(Set && (1 / setToArray(new Set([,-0]))[1]) == INFINITY) ? noop : function(values) { - return new Set(values); - }; - - /** - * Creates a `_.toPairs` or `_.toPairsIn` function. - * - * @private - * @param {Function} keysFunc The function to get the keys of a given object. - * @returns {Function} Returns the new pairs function. - */ - function createToPairs(keysFunc) { - return function(object) { - var tag = getTag(object); - if (tag == mapTag) { - return mapToArray(object); - } - if (tag == setTag) { - return setToPairs(object); - } - return baseToPairs(object, keysFunc(object)); - }; - } - - /** - * Creates a function that either curries or invokes `func` with optional - * `this` binding and partially applied arguments. - * - * @private - * @param {Function|string} func The function or method name to wrap. - * @param {number} bitmask The bitmask flags. - * 1 - `_.bind` - * 2 - `_.bindKey` - * 4 - `_.curry` or `_.curryRight` of a bound function - * 8 - `_.curry` - * 16 - `_.curryRight` - * 32 - `_.partial` - * 64 - `_.partialRight` - * 128 - `_.rearg` - * 256 - `_.ary` - * 512 - `_.flip` - * @param {*} [thisArg] The `this` binding of `func`. - * @param {Array} [partials] The arguments to be partially applied. - * @param {Array} [holders] The `partials` placeholder indexes. - * @param {Array} [argPos] The argument positions of the new function. - * @param {number} [ary] The arity cap of `func`. - * @param {number} [arity] The arity of `func`. - * @returns {Function} Returns the new wrapped function. - */ - function createWrap(func, bitmask, thisArg, partials, holders, argPos, ary, arity) { - var isBindKey = bitmask & WRAP_BIND_KEY_FLAG; - if (!isBindKey && typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - var length = partials ? partials.length : 0; - if (!length) { - bitmask &= ~(WRAP_PARTIAL_FLAG | WRAP_PARTIAL_RIGHT_FLAG); - partials = holders = undefined; - } - ary = ary === undefined ? ary : nativeMax(toInteger(ary), 0); - arity = arity === undefined ? arity : toInteger(arity); - length -= holders ? holders.length : 0; - - if (bitmask & WRAP_PARTIAL_RIGHT_FLAG) { - var partialsRight = partials, - holdersRight = holders; - - partials = holders = undefined; - } - var data = isBindKey ? undefined : getData(func); - - var newData = [ - func, bitmask, thisArg, partials, holders, partialsRight, holdersRight, - argPos, ary, arity - ]; - - if (data) { - mergeData(newData, data); - } - func = newData[0]; - bitmask = newData[1]; - thisArg = newData[2]; - partials = newData[3]; - holders = newData[4]; - arity = newData[9] = newData[9] === undefined - ? (isBindKey ? 0 : func.length) - : nativeMax(newData[9] - length, 0); - - if (!arity && bitmask & (WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG)) { - bitmask &= ~(WRAP_CURRY_FLAG | WRAP_CURRY_RIGHT_FLAG); - } - if (!bitmask || bitmask == WRAP_BIND_FLAG) { - var result = createBind(func, bitmask, thisArg); - } else if (bitmask == WRAP_CURRY_FLAG || bitmask == WRAP_CURRY_RIGHT_FLAG) { - result = createCurry(func, bitmask, arity); - } else if ((bitmask == WRAP_PARTIAL_FLAG || bitmask == (WRAP_BIND_FLAG | WRAP_PARTIAL_FLAG)) && !holders.length) { - result = createPartial(func, bitmask, thisArg, partials); - } else { - result = createHybrid.apply(undefined, newData); - } - var setter = data ? baseSetData : setData; - return setWrapToString(setter(result, newData), func, bitmask); - } - - /** - * Used by `_.defaults` to customize its `_.assignIn` use to assign properties - * of source objects to the destination object for all destination properties - * that resolve to `undefined`. - * - * @private - * @param {*} objValue The destination value. - * @param {*} srcValue The source value. - * @param {string} key The key of the property to assign. - * @param {Object} object The parent object of `objValue`. - * @returns {*} Returns the value to assign. - */ - function customDefaultsAssignIn(objValue, srcValue, key, object) { - if (objValue === undefined || - (eq(objValue, objectProto[key]) && !hasOwnProperty.call(object, key))) { - return srcValue; - } - return objValue; - } - - /** - * Used by `_.defaultsDeep` to customize its `_.merge` use to merge source - * objects into destination objects that are passed thru. - * - * @private - * @param {*} objValue The destination value. - * @param {*} srcValue The source value. - * @param {string} key The key of the property to merge. - * @param {Object} object The parent object of `objValue`. - * @param {Object} source The parent object of `srcValue`. - * @param {Object} [stack] Tracks traversed source values and their merged - * counterparts. - * @returns {*} Returns the value to assign. - */ - function customDefaultsMerge(objValue, srcValue, key, object, source, stack) { - if (isObject(objValue) && isObject(srcValue)) { - // Recursively merge objects and arrays (susceptible to call stack limits). - stack.set(srcValue, objValue); - baseMerge(objValue, srcValue, undefined, customDefaultsMerge, stack); - stack['delete'](srcValue); - } - return objValue; - } - - /** - * Used by `_.omit` to customize its `_.cloneDeep` use to only clone plain - * objects. - * - * @private - * @param {*} value The value to inspect. - * @param {string} key The key of the property to inspect. - * @returns {*} Returns the uncloned value or `undefined` to defer cloning to `_.cloneDeep`. - */ - function customOmitClone(value) { - return isPlainObject(value) ? undefined : value; - } - - /** - * A specialized version of `baseIsEqualDeep` for arrays with support for - * partial deep comparisons. - * - * @private - * @param {Array} array The array to compare. - * @param {Array} other The other array to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} stack Tracks traversed `array` and `other` objects. - * @returns {boolean} Returns `true` if the arrays are equivalent, else `false`. - */ - function equalArrays(array, other, bitmask, customizer, equalFunc, stack) { - var isPartial = bitmask & COMPARE_PARTIAL_FLAG, - arrLength = array.length, - othLength = other.length; - - if (arrLength != othLength && !(isPartial && othLength > arrLength)) { - return false; - } - // Assume cyclic values are equal. - var stacked = stack.get(array); - if (stacked && stack.get(other)) { - return stacked == other; - } - var index = -1, - result = true, - seen = (bitmask & COMPARE_UNORDERED_FLAG) ? new SetCache : undefined; - - stack.set(array, other); - stack.set(other, array); - - // Ignore non-index properties. - while (++index < arrLength) { - var arrValue = array[index], - othValue = other[index]; - - if (customizer) { - var compared = isPartial - ? customizer(othValue, arrValue, index, other, array, stack) - : customizer(arrValue, othValue, index, array, other, stack); - } - if (compared !== undefined) { - if (compared) { - continue; - } - result = false; - break; - } - // Recursively compare arrays (susceptible to call stack limits). - if (seen) { - if (!arraySome(other, function(othValue, othIndex) { - if (!cacheHas(seen, othIndex) && - (arrValue === othValue || equalFunc(arrValue, othValue, bitmask, customizer, stack))) { - return seen.push(othIndex); - } - })) { - result = false; - break; - } - } else if (!( - arrValue === othValue || - equalFunc(arrValue, othValue, bitmask, customizer, stack) - )) { - result = false; - break; - } - } - stack['delete'](array); - stack['delete'](other); - return result; - } - - /** - * A specialized version of `baseIsEqualDeep` for comparing objects of - * the same `toStringTag`. - * - * **Note:** This function only supports comparing values with tags of - * `Boolean`, `Date`, `Error`, `Number`, `RegExp`, or `String`. - * - * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {string} tag The `toStringTag` of the objects to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} stack Tracks traversed `object` and `other` objects. - * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. - */ - function equalByTag(object, other, tag, bitmask, customizer, equalFunc, stack) { - switch (tag) { - case dataViewTag: - if ((object.byteLength != other.byteLength) || - (object.byteOffset != other.byteOffset)) { - return false; - } - object = object.buffer; - other = other.buffer; - - case arrayBufferTag: - if ((object.byteLength != other.byteLength) || - !equalFunc(new Uint8Array(object), new Uint8Array(other))) { - return false; - } - return true; - - case boolTag: - case dateTag: - case numberTag: - // Coerce booleans to `1` or `0` and dates to milliseconds. - // Invalid dates are coerced to `NaN`. - return eq(+object, +other); - - case errorTag: - return object.name == other.name && object.message == other.message; - - case regexpTag: - case stringTag: - // Coerce regexes to strings and treat strings, primitives and objects, - // as equal. See http://www.ecma-international.org/ecma-262/7.0/#sec-regexp.prototype.tostring - // for more details. - return object == (other + ''); - - case mapTag: - var convert = mapToArray; - - case setTag: - var isPartial = bitmask & COMPARE_PARTIAL_FLAG; - convert || (convert = setToArray); - - if (object.size != other.size && !isPartial) { - return false; - } - // Assume cyclic values are equal. - var stacked = stack.get(object); - if (stacked) { - return stacked == other; - } - bitmask |= COMPARE_UNORDERED_FLAG; - - // Recursively compare objects (susceptible to call stack limits). - stack.set(object, other); - var result = equalArrays(convert(object), convert(other), bitmask, customizer, equalFunc, stack); - stack['delete'](object); - return result; - - case symbolTag: - if (symbolValueOf) { - return symbolValueOf.call(object) == symbolValueOf.call(other); - } - } - return false; - } - - /** - * A specialized version of `baseIsEqualDeep` for objects with support for - * partial deep comparisons. - * - * @private - * @param {Object} object The object to compare. - * @param {Object} other The other object to compare. - * @param {number} bitmask The bitmask flags. See `baseIsEqual` for more details. - * @param {Function} customizer The function to customize comparisons. - * @param {Function} equalFunc The function to determine equivalents of values. - * @param {Object} stack Tracks traversed `object` and `other` objects. - * @returns {boolean} Returns `true` if the objects are equivalent, else `false`. - */ - function equalObjects(object, other, bitmask, customizer, equalFunc, stack) { - var isPartial = bitmask & COMPARE_PARTIAL_FLAG, - objProps = getAllKeys(object), - objLength = objProps.length, - othProps = getAllKeys(other), - othLength = othProps.length; - - if (objLength != othLength && !isPartial) { - return false; - } - var index = objLength; - while (index--) { - var key = objProps[index]; - if (!(isPartial ? key in other : hasOwnProperty.call(other, key))) { - return false; - } - } - // Assume cyclic values are equal. - var stacked = stack.get(object); - if (stacked && stack.get(other)) { - return stacked == other; - } - var result = true; - stack.set(object, other); - stack.set(other, object); - - var skipCtor = isPartial; - while (++index < objLength) { - key = objProps[index]; - var objValue = object[key], - othValue = other[key]; - - if (customizer) { - var compared = isPartial - ? customizer(othValue, objValue, key, other, object, stack) - : customizer(objValue, othValue, key, object, other, stack); - } - // Recursively compare objects (susceptible to call stack limits). - if (!(compared === undefined - ? (objValue === othValue || equalFunc(objValue, othValue, bitmask, customizer, stack)) - : compared - )) { - result = false; - break; - } - skipCtor || (skipCtor = key == 'constructor'); - } - if (result && !skipCtor) { - var objCtor = object.constructor, - othCtor = other.constructor; - - // Non `Object` object instances with different constructors are not equal. - if (objCtor != othCtor && - ('constructor' in object && 'constructor' in other) && - !(typeof objCtor == 'function' && objCtor instanceof objCtor && - typeof othCtor == 'function' && othCtor instanceof othCtor)) { - result = false; - } - } - stack['delete'](object); - stack['delete'](other); - return result; - } - - /** - * A specialized version of `baseRest` which flattens the rest array. - * - * @private - * @param {Function} func The function to apply a rest parameter to. - * @returns {Function} Returns the new function. - */ - function flatRest(func) { - return setToString(overRest(func, undefined, flatten), func + ''); - } - - /** - * Creates an array of own enumerable property names and symbols of `object`. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names and symbols. - */ - function getAllKeys(object) { - return baseGetAllKeys(object, keys, getSymbols); - } - - /** - * Creates an array of own and inherited enumerable property names and - * symbols of `object`. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names and symbols. - */ - function getAllKeysIn(object) { - return baseGetAllKeys(object, keysIn, getSymbolsIn); - } - - /** - * Gets metadata for `func`. - * - * @private - * @param {Function} func The function to query. - * @returns {*} Returns the metadata for `func`. - */ - var getData = !metaMap ? noop : function(func) { - return metaMap.get(func); - }; - - /** - * Gets the name of `func`. - * - * @private - * @param {Function} func The function to query. - * @returns {string} Returns the function name. - */ - function getFuncName(func) { - var result = (func.name + ''), - array = realNames[result], - length = hasOwnProperty.call(realNames, result) ? array.length : 0; - - while (length--) { - var data = array[length], - otherFunc = data.func; - if (otherFunc == null || otherFunc == func) { - return data.name; - } - } - return result; - } - - /** - * Gets the argument placeholder value for `func`. - * - * @private - * @param {Function} func The function to inspect. - * @returns {*} Returns the placeholder value. - */ - function getHolder(func) { - var object = hasOwnProperty.call(lodash, 'placeholder') ? lodash : func; - return object.placeholder; - } - - /** - * Gets the appropriate "iteratee" function. If `_.iteratee` is customized, - * this function returns the custom method, otherwise it returns `baseIteratee`. - * If arguments are provided, the chosen function is invoked with them and - * its result is returned. - * - * @private - * @param {*} [value] The value to convert to an iteratee. - * @param {number} [arity] The arity of the created iteratee. - * @returns {Function} Returns the chosen function or its result. - */ - function getIteratee() { - var result = lodash.iteratee || iteratee; - result = result === iteratee ? baseIteratee : result; - return arguments.length ? result(arguments[0], arguments[1]) : result; - } - - /** - * Gets the data for `map`. - * - * @private - * @param {Object} map The map to query. - * @param {string} key The reference key. - * @returns {*} Returns the map data. - */ - function getMapData(map, key) { - var data = map.__data__; - return isKeyable(key) - ? data[typeof key == 'string' ? 'string' : 'hash'] - : data.map; - } - - /** - * Gets the property names, values, and compare flags of `object`. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the match data of `object`. - */ - function getMatchData(object) { - var result = keys(object), - length = result.length; - - while (length--) { - var key = result[length], - value = object[key]; - - result[length] = [key, value, isStrictComparable(value)]; - } - return result; - } - - /** - * Gets the native function at `key` of `object`. - * - * @private - * @param {Object} object The object to query. - * @param {string} key The key of the method to get. - * @returns {*} Returns the function if it's native, else `undefined`. - */ - function getNative(object, key) { - var value = getValue(object, key); - return baseIsNative(value) ? value : undefined; - } - - /** - * A specialized version of `baseGetTag` which ignores `Symbol.toStringTag` values. - * - * @private - * @param {*} value The value to query. - * @returns {string} Returns the raw `toStringTag`. - */ - function getRawTag(value) { - var isOwn = hasOwnProperty.call(value, symToStringTag), - tag = value[symToStringTag]; - - try { - value[symToStringTag] = undefined; - var unmasked = true; - } catch (e) {} - - var result = nativeObjectToString.call(value); - if (unmasked) { - if (isOwn) { - value[symToStringTag] = tag; - } else { - delete value[symToStringTag]; - } - } - return result; - } - - /** - * Creates an array of the own enumerable symbols of `object`. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of symbols. - */ - var getSymbols = !nativeGetSymbols ? stubArray : function(object) { - if (object == null) { - return []; - } - object = Object(object); - return arrayFilter(nativeGetSymbols(object), function(symbol) { - return propertyIsEnumerable.call(object, symbol); - }); - }; - - /** - * Creates an array of the own and inherited enumerable symbols of `object`. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of symbols. - */ - var getSymbolsIn = !nativeGetSymbols ? stubArray : function(object) { - var result = []; - while (object) { - arrayPush(result, getSymbols(object)); - object = getPrototype(object); - } - return result; - }; - - /** - * Gets the `toStringTag` of `value`. - * - * @private - * @param {*} value The value to query. - * @returns {string} Returns the `toStringTag`. - */ - var getTag = baseGetTag; - - // Fallback for data views, maps, sets, and weak maps in IE 11 and promises in Node.js < 6. - if ((DataView && getTag(new DataView(new ArrayBuffer(1))) != dataViewTag) || - (Map && getTag(new Map) != mapTag) || - (Promise && getTag(Promise.resolve()) != promiseTag) || - (Set && getTag(new Set) != setTag) || - (WeakMap && getTag(new WeakMap) != weakMapTag)) { - getTag = function(value) { - var result = baseGetTag(value), - Ctor = result == objectTag ? value.constructor : undefined, - ctorString = Ctor ? toSource(Ctor) : ''; - - if (ctorString) { - switch (ctorString) { - case dataViewCtorString: return dataViewTag; - case mapCtorString: return mapTag; - case promiseCtorString: return promiseTag; - case setCtorString: return setTag; - case weakMapCtorString: return weakMapTag; - } - } - return result; - }; - } - - /** - * Gets the view, applying any `transforms` to the `start` and `end` positions. - * - * @private - * @param {number} start The start of the view. - * @param {number} end The end of the view. - * @param {Array} transforms The transformations to apply to the view. - * @returns {Object} Returns an object containing the `start` and `end` - * positions of the view. - */ - function getView(start, end, transforms) { - var index = -1, - length = transforms.length; - - while (++index < length) { - var data = transforms[index], - size = data.size; - - switch (data.type) { - case 'drop': start += size; break; - case 'dropRight': end -= size; break; - case 'take': end = nativeMin(end, start + size); break; - case 'takeRight': start = nativeMax(start, end - size); break; - } - } - return { 'start': start, 'end': end }; - } - - /** - * Extracts wrapper details from the `source` body comment. - * - * @private - * @param {string} source The source to inspect. - * @returns {Array} Returns the wrapper details. - */ - function getWrapDetails(source) { - var match = source.match(reWrapDetails); - return match ? match[1].split(reSplitDetails) : []; - } - - /** - * Checks if `path` exists on `object`. - * - * @private - * @param {Object} object The object to query. - * @param {Array|string} path The path to check. - * @param {Function} hasFunc The function to check properties. - * @returns {boolean} Returns `true` if `path` exists, else `false`. - */ - function hasPath(object, path, hasFunc) { - path = castPath(path, object); - - var index = -1, - length = path.length, - result = false; - - while (++index < length) { - var key = toKey(path[index]); - if (!(result = object != null && hasFunc(object, key))) { - break; - } - object = object[key]; - } - if (result || ++index != length) { - return result; - } - length = object == null ? 0 : object.length; - return !!length && isLength(length) && isIndex(key, length) && - (isArray(object) || isArguments(object)); - } - - /** - * Initializes an array clone. - * - * @private - * @param {Array} array The array to clone. - * @returns {Array} Returns the initialized clone. - */ - function initCloneArray(array) { - var length = array.length, - result = new array.constructor(length); - - // Add properties assigned by `RegExp#exec`. - if (length && typeof array[0] == 'string' && hasOwnProperty.call(array, 'index')) { - result.index = array.index; - result.input = array.input; - } - return result; - } - - /** - * Initializes an object clone. - * - * @private - * @param {Object} object The object to clone. - * @returns {Object} Returns the initialized clone. - */ - function initCloneObject(object) { - return (typeof object.constructor == 'function' && !isPrototype(object)) - ? baseCreate(getPrototype(object)) - : {}; - } - - /** - * Initializes an object clone based on its `toStringTag`. - * - * **Note:** This function only supports cloning values with tags of - * `Boolean`, `Date`, `Error`, `Map`, `Number`, `RegExp`, `Set`, or `String`. - * - * @private - * @param {Object} object The object to clone. - * @param {string} tag The `toStringTag` of the object to clone. - * @param {boolean} [isDeep] Specify a deep clone. - * @returns {Object} Returns the initialized clone. - */ - function initCloneByTag(object, tag, isDeep) { - var Ctor = object.constructor; - switch (tag) { - case arrayBufferTag: - return cloneArrayBuffer(object); - - case boolTag: - case dateTag: - return new Ctor(+object); - - case dataViewTag: - return cloneDataView(object, isDeep); - - case float32Tag: case float64Tag: - case int8Tag: case int16Tag: case int32Tag: - case uint8Tag: case uint8ClampedTag: case uint16Tag: case uint32Tag: - return cloneTypedArray(object, isDeep); - - case mapTag: - return new Ctor; - - case numberTag: - case stringTag: - return new Ctor(object); - - case regexpTag: - return cloneRegExp(object); - - case setTag: - return new Ctor; - - case symbolTag: - return cloneSymbol(object); - } - } - - /** - * Inserts wrapper `details` in a comment at the top of the `source` body. - * - * @private - * @param {string} source The source to modify. - * @returns {Array} details The details to insert. - * @returns {string} Returns the modified source. - */ - function insertWrapDetails(source, details) { - var length = details.length; - if (!length) { - return source; - } - var lastIndex = length - 1; - details[lastIndex] = (length > 1 ? '& ' : '') + details[lastIndex]; - details = details.join(length > 2 ? ', ' : ' '); - return source.replace(reWrapComment, '{\n/* [wrapped with ' + details + '] */\n'); - } - - /** - * Checks if `value` is a flattenable `arguments` object or array. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is flattenable, else `false`. - */ - function isFlattenable(value) { - return isArray(value) || isArguments(value) || - !!(spreadableSymbol && value && value[spreadableSymbol]); - } - - /** - * Checks if `value` is a valid array-like index. - * - * @private - * @param {*} value The value to check. - * @param {number} [length=MAX_SAFE_INTEGER] The upper bounds of a valid index. - * @returns {boolean} Returns `true` if `value` is a valid index, else `false`. - */ - function isIndex(value, length) { - var type = typeof value; - length = length == null ? MAX_SAFE_INTEGER : length; - - return !!length && - (type == 'number' || - (type != 'symbol' && reIsUint.test(value))) && - (value > -1 && value % 1 == 0 && value < length); - } - - /** - * Checks if the given arguments are from an iteratee call. - * - * @private - * @param {*} value The potential iteratee value argument. - * @param {*} index The potential iteratee index or key argument. - * @param {*} object The potential iteratee object argument. - * @returns {boolean} Returns `true` if the arguments are from an iteratee call, - * else `false`. - */ - function isIterateeCall(value, index, object) { - if (!isObject(object)) { - return false; - } - var type = typeof index; - if (type == 'number' - ? (isArrayLike(object) && isIndex(index, object.length)) - : (type == 'string' && index in object) - ) { - return eq(object[index], value); - } - return false; - } - - /** - * Checks if `value` is a property name and not a property path. - * - * @private - * @param {*} value The value to check. - * @param {Object} [object] The object to query keys on. - * @returns {boolean} Returns `true` if `value` is a property name, else `false`. - */ - function isKey(value, object) { - if (isArray(value)) { - return false; - } - var type = typeof value; - if (type == 'number' || type == 'symbol' || type == 'boolean' || - value == null || isSymbol(value)) { - return true; - } - return reIsPlainProp.test(value) || !reIsDeepProp.test(value) || - (object != null && value in Object(object)); - } - - /** - * Checks if `value` is suitable for use as unique object key. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is suitable, else `false`. - */ - function isKeyable(value) { - var type = typeof value; - return (type == 'string' || type == 'number' || type == 'symbol' || type == 'boolean') - ? (value !== '__proto__') - : (value === null); - } - - /** - * Checks if `func` has a lazy counterpart. - * - * @private - * @param {Function} func The function to check. - * @returns {boolean} Returns `true` if `func` has a lazy counterpart, - * else `false`. - */ - function isLaziable(func) { - var funcName = getFuncName(func), - other = lodash[funcName]; - - if (typeof other != 'function' || !(funcName in LazyWrapper.prototype)) { - return false; - } - if (func === other) { - return true; - } - var data = getData(other); - return !!data && func === data[0]; - } - - /** - * Checks if `func` has its source masked. - * - * @private - * @param {Function} func The function to check. - * @returns {boolean} Returns `true` if `func` is masked, else `false`. - */ - function isMasked(func) { - return !!maskSrcKey && (maskSrcKey in func); - } - - /** - * Checks if `func` is capable of being masked. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `func` is maskable, else `false`. - */ - var isMaskable = coreJsData ? isFunction : stubFalse; - - /** - * Checks if `value` is likely a prototype object. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a prototype, else `false`. - */ - function isPrototype(value) { - var Ctor = value && value.constructor, - proto = (typeof Ctor == 'function' && Ctor.prototype) || objectProto; - - return value === proto; - } - - /** - * Checks if `value` is suitable for strict equality comparisons, i.e. `===`. - * - * @private - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` if suitable for strict - * equality comparisons, else `false`. - */ - function isStrictComparable(value) { - return value === value && !isObject(value); - } - - /** - * A specialized version of `matchesProperty` for source values suitable - * for strict equality comparisons, i.e. `===`. - * - * @private - * @param {string} key The key of the property to get. - * @param {*} srcValue The value to match. - * @returns {Function} Returns the new spec function. - */ - function matchesStrictComparable(key, srcValue) { - return function(object) { - if (object == null) { - return false; - } - return object[key] === srcValue && - (srcValue !== undefined || (key in Object(object))); - }; - } - - /** - * A specialized version of `_.memoize` which clears the memoized function's - * cache when it exceeds `MAX_MEMOIZE_SIZE`. - * - * @private - * @param {Function} func The function to have its output memoized. - * @returns {Function} Returns the new memoized function. - */ - function memoizeCapped(func) { - var result = memoize(func, function(key) { - if (cache.size === MAX_MEMOIZE_SIZE) { - cache.clear(); - } - return key; - }); - - var cache = result.cache; - return result; - } - - /** - * Merges the function metadata of `source` into `data`. - * - * Merging metadata reduces the number of wrappers used to invoke a function. - * This is possible because methods like `_.bind`, `_.curry`, and `_.partial` - * may be applied regardless of execution order. Methods like `_.ary` and - * `_.rearg` modify function arguments, making the order in which they are - * executed important, preventing the merging of metadata. However, we make - * an exception for a safe combined case where curried functions have `_.ary` - * and or `_.rearg` applied. - * - * @private - * @param {Array} data The destination metadata. - * @param {Array} source The source metadata. - * @returns {Array} Returns `data`. - */ - function mergeData(data, source) { - var bitmask = data[1], - srcBitmask = source[1], - newBitmask = bitmask | srcBitmask, - isCommon = newBitmask < (WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG | WRAP_ARY_FLAG); - - var isCombo = - ((srcBitmask == WRAP_ARY_FLAG) && (bitmask == WRAP_CURRY_FLAG)) || - ((srcBitmask == WRAP_ARY_FLAG) && (bitmask == WRAP_REARG_FLAG) && (data[7].length <= source[8])) || - ((srcBitmask == (WRAP_ARY_FLAG | WRAP_REARG_FLAG)) && (source[7].length <= source[8]) && (bitmask == WRAP_CURRY_FLAG)); - - // Exit early if metadata can't be merged. - if (!(isCommon || isCombo)) { - return data; - } - // Use source `thisArg` if available. - if (srcBitmask & WRAP_BIND_FLAG) { - data[2] = source[2]; - // Set when currying a bound function. - newBitmask |= bitmask & WRAP_BIND_FLAG ? 0 : WRAP_CURRY_BOUND_FLAG; - } - // Compose partial arguments. - var value = source[3]; - if (value) { - var partials = data[3]; - data[3] = partials ? composeArgs(partials, value, source[4]) : value; - data[4] = partials ? replaceHolders(data[3], PLACEHOLDER) : source[4]; - } - // Compose partial right arguments. - value = source[5]; - if (value) { - partials = data[5]; - data[5] = partials ? composeArgsRight(partials, value, source[6]) : value; - data[6] = partials ? replaceHolders(data[5], PLACEHOLDER) : source[6]; - } - // Use source `argPos` if available. - value = source[7]; - if (value) { - data[7] = value; - } - // Use source `ary` if it's smaller. - if (srcBitmask & WRAP_ARY_FLAG) { - data[8] = data[8] == null ? source[8] : nativeMin(data[8], source[8]); - } - // Use source `arity` if one is not provided. - if (data[9] == null) { - data[9] = source[9]; - } - // Use source `func` and merge bitmasks. - data[0] = source[0]; - data[1] = newBitmask; - - return data; - } - - /** - * This function is like - * [`Object.keys`](http://ecma-international.org/ecma-262/7.0/#sec-object.keys) - * except that it includes inherited enumerable properties. - * - * @private - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. - */ - function nativeKeysIn(object) { - var result = []; - if (object != null) { - for (var key in Object(object)) { - result.push(key); - } - } - return result; - } - - /** - * Converts `value` to a string using `Object.prototype.toString`. - * - * @private - * @param {*} value The value to convert. - * @returns {string} Returns the converted string. - */ - function objectToString(value) { - return nativeObjectToString.call(value); - } - - /** - * A specialized version of `baseRest` which transforms the rest array. - * - * @private - * @param {Function} func The function to apply a rest parameter to. - * @param {number} [start=func.length-1] The start position of the rest parameter. - * @param {Function} transform The rest array transform. - * @returns {Function} Returns the new function. - */ - function overRest(func, start, transform) { - start = nativeMax(start === undefined ? (func.length - 1) : start, 0); - return function() { - var args = arguments, - index = -1, - length = nativeMax(args.length - start, 0), - array = Array(length); - - while (++index < length) { - array[index] = args[start + index]; - } - index = -1; - var otherArgs = Array(start + 1); - while (++index < start) { - otherArgs[index] = args[index]; - } - otherArgs[start] = transform(array); - return apply(func, this, otherArgs); - }; - } - - /** - * Gets the parent value at `path` of `object`. - * - * @private - * @param {Object} object The object to query. - * @param {Array} path The path to get the parent value of. - * @returns {*} Returns the parent value. - */ - function parent(object, path) { - return path.length < 2 ? object : baseGet(object, baseSlice(path, 0, -1)); - } - - /** - * Reorder `array` according to the specified indexes where the element at - * the first index is assigned as the first element, the element at - * the second index is assigned as the second element, and so on. - * - * @private - * @param {Array} array The array to reorder. - * @param {Array} indexes The arranged array indexes. - * @returns {Array} Returns `array`. - */ - function reorder(array, indexes) { - var arrLength = array.length, - length = nativeMin(indexes.length, arrLength), - oldArray = copyArray(array); - - while (length--) { - var index = indexes[length]; - array[length] = isIndex(index, arrLength) ? oldArray[index] : undefined; - } - return array; - } - - /** - * Gets the value at `key`, unless `key` is "__proto__" or "constructor". - * - * @private - * @param {Object} object The object to query. - * @param {string} key The key of the property to get. - * @returns {*} Returns the property value. - */ - function safeGet(object, key) { - if (key === 'constructor' && typeof object[key] === 'function') { - return; - } - - if (key == '__proto__') { - return; - } - - return object[key]; - } - - /** - * Sets metadata for `func`. - * - * **Note:** If this function becomes hot, i.e. is invoked a lot in a short - * period of time, it will trip its breaker and transition to an identity - * function to avoid garbage collection pauses in V8. See - * [V8 issue 2070](https://bugs.chromium.org/p/v8/issues/detail?id=2070) - * for more details. - * - * @private - * @param {Function} func The function to associate metadata with. - * @param {*} data The metadata. - * @returns {Function} Returns `func`. - */ - var setData = shortOut(baseSetData); - - /** - * A simple wrapper around the global [`setTimeout`](https://mdn.io/setTimeout). - * - * @private - * @param {Function} func The function to delay. - * @param {number} wait The number of milliseconds to delay invocation. - * @returns {number|Object} Returns the timer id or timeout object. - */ - var setTimeout = ctxSetTimeout || function(func, wait) { - return root.setTimeout(func, wait); - }; - - /** - * Sets the `toString` method of `func` to return `string`. - * - * @private - * @param {Function} func The function to modify. - * @param {Function} string The `toString` result. - * @returns {Function} Returns `func`. - */ - var setToString = shortOut(baseSetToString); - - /** - * Sets the `toString` method of `wrapper` to mimic the source of `reference` - * with wrapper details in a comment at the top of the source body. - * - * @private - * @param {Function} wrapper The function to modify. - * @param {Function} reference The reference function. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @returns {Function} Returns `wrapper`. - */ - function setWrapToString(wrapper, reference, bitmask) { - var source = (reference + ''); - return setToString(wrapper, insertWrapDetails(source, updateWrapDetails(getWrapDetails(source), bitmask))); - } - - /** - * Creates a function that'll short out and invoke `identity` instead - * of `func` when it's called `HOT_COUNT` or more times in `HOT_SPAN` - * milliseconds. - * - * @private - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new shortable function. - */ - function shortOut(func) { - var count = 0, - lastCalled = 0; - - return function() { - var stamp = nativeNow(), - remaining = HOT_SPAN - (stamp - lastCalled); - - lastCalled = stamp; - if (remaining > 0) { - if (++count >= HOT_COUNT) { - return arguments[0]; - } - } else { - count = 0; - } - return func.apply(undefined, arguments); - }; - } - - /** - * A specialized version of `_.shuffle` which mutates and sets the size of `array`. - * - * @private - * @param {Array} array The array to shuffle. - * @param {number} [size=array.length] The size of `array`. - * @returns {Array} Returns `array`. - */ - function shuffleSelf(array, size) { - var index = -1, - length = array.length, - lastIndex = length - 1; - - size = size === undefined ? length : size; - while (++index < size) { - var rand = baseRandom(index, lastIndex), - value = array[rand]; - - array[rand] = array[index]; - array[index] = value; - } - array.length = size; - return array; - } - - /** - * Converts `string` to a property path array. - * - * @private - * @param {string} string The string to convert. - * @returns {Array} Returns the property path array. - */ - var stringToPath = memoizeCapped(function(string) { - var result = []; - if (string.charCodeAt(0) === 46 /* . */) { - result.push(''); - } - string.replace(rePropName, function(match, number, quote, subString) { - result.push(quote ? subString.replace(reEscapeChar, '$1') : (number || match)); - }); - return result; - }); - - /** - * Converts `value` to a string key if it's not a string or symbol. - * - * @private - * @param {*} value The value to inspect. - * @returns {string|symbol} Returns the key. - */ - function toKey(value) { - if (typeof value == 'string' || isSymbol(value)) { - return value; - } - var result = (value + ''); - return (result == '0' && (1 / value) == -INFINITY) ? '-0' : result; - } - - /** - * Converts `func` to its source code. - * - * @private - * @param {Function} func The function to convert. - * @returns {string} Returns the source code. - */ - function toSource(func) { - if (func != null) { - try { - return funcToString.call(func); - } catch (e) {} - try { - return (func + ''); - } catch (e) {} - } - return ''; - } - - /** - * Updates wrapper `details` based on `bitmask` flags. - * - * @private - * @returns {Array} details The details to modify. - * @param {number} bitmask The bitmask flags. See `createWrap` for more details. - * @returns {Array} Returns `details`. - */ - function updateWrapDetails(details, bitmask) { - arrayEach(wrapFlags, function(pair) { - var value = '_.' + pair[0]; - if ((bitmask & pair[1]) && !arrayIncludes(details, value)) { - details.push(value); - } - }); - return details.sort(); - } - - /** - * Creates a clone of `wrapper`. - * - * @private - * @param {Object} wrapper The wrapper to clone. - * @returns {Object} Returns the cloned wrapper. - */ - function wrapperClone(wrapper) { - if (wrapper instanceof LazyWrapper) { - return wrapper.clone(); - } - var result = new LodashWrapper(wrapper.__wrapped__, wrapper.__chain__); - result.__actions__ = copyArray(wrapper.__actions__); - result.__index__ = wrapper.__index__; - result.__values__ = wrapper.__values__; - return result; - } - - /*------------------------------------------------------------------------*/ - - /** - * Creates an array of elements split into groups the length of `size`. - * If `array` can't be split evenly, the final chunk will be the remaining - * elements. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to process. - * @param {number} [size=1] The length of each chunk - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the new array of chunks. - * @example - * - * _.chunk(['a', 'b', 'c', 'd'], 2); - * // => [['a', 'b'], ['c', 'd']] - * - * _.chunk(['a', 'b', 'c', 'd'], 3); - * // => [['a', 'b', 'c'], ['d']] - */ - function chunk(array, size, guard) { - if ((guard ? isIterateeCall(array, size, guard) : size === undefined)) { - size = 1; - } else { - size = nativeMax(toInteger(size), 0); - } - var length = array == null ? 0 : array.length; - if (!length || size < 1) { - return []; - } - var index = 0, - resIndex = 0, - result = Array(nativeCeil(length / size)); - - while (index < length) { - result[resIndex++] = baseSlice(array, index, (index += size)); - } - return result; - } - - /** - * Creates an array with all falsey values removed. The values `false`, `null`, - * `0`, `""`, `undefined`, and `NaN` are falsey. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to compact. - * @returns {Array} Returns the new array of filtered values. - * @example - * - * _.compact([0, 1, false, 2, '', 3]); - * // => [1, 2, 3] - */ - function compact(array) { - var index = -1, - length = array == null ? 0 : array.length, - resIndex = 0, - result = []; - - while (++index < length) { - var value = array[index]; - if (value) { - result[resIndex++] = value; - } - } - return result; - } - - /** - * Creates a new array concatenating `array` with any additional arrays - * and/or values. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to concatenate. - * @param {...*} [values] The values to concatenate. - * @returns {Array} Returns the new concatenated array. - * @example - * - * var array = [1]; - * var other = _.concat(array, 2, [3], [[4]]); - * - * console.log(other); - * // => [1, 2, 3, [4]] - * - * console.log(array); - * // => [1] - */ - function concat() { - var length = arguments.length; - if (!length) { - return []; - } - var args = Array(length - 1), - array = arguments[0], - index = length; - - while (index--) { - args[index - 1] = arguments[index]; - } - return arrayPush(isArray(array) ? copyArray(array) : [array], baseFlatten(args, 1)); - } - - /** - * Creates an array of `array` values not included in the other given arrays - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. The order and references of result values are - * determined by the first array. - * - * **Note:** Unlike `_.pullAll`, this method returns a new array. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {...Array} [values] The values to exclude. - * @returns {Array} Returns the new array of filtered values. - * @see _.without, _.xor - * @example - * - * _.difference([2, 1], [2, 3]); - * // => [1] - */ - var difference = baseRest(function(array, values) { - return isArrayLikeObject(array) - ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true)) - : []; - }); - - /** - * This method is like `_.difference` except that it accepts `iteratee` which - * is invoked for each element of `array` and `values` to generate the criterion - * by which they're compared. The order and references of result values are - * determined by the first array. The iteratee is invoked with one argument: - * (value). - * - * **Note:** Unlike `_.pullAllBy`, this method returns a new array. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {...Array} [values] The values to exclude. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of filtered values. - * @example - * - * _.differenceBy([2.1, 1.2], [2.3, 3.4], Math.floor); - * // => [1.2] - * - * // The `_.property` iteratee shorthand. - * _.differenceBy([{ 'x': 2 }, { 'x': 1 }], [{ 'x': 1 }], 'x'); - * // => [{ 'x': 2 }] - */ - var differenceBy = baseRest(function(array, values) { - var iteratee = last(values); - if (isArrayLikeObject(iteratee)) { - iteratee = undefined; - } - return isArrayLikeObject(array) - ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true), getIteratee(iteratee, 2)) - : []; - }); - - /** - * This method is like `_.difference` except that it accepts `comparator` - * which is invoked to compare elements of `array` to `values`. The order and - * references of result values are determined by the first array. The comparator - * is invoked with two arguments: (arrVal, othVal). - * - * **Note:** Unlike `_.pullAllWith`, this method returns a new array. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {...Array} [values] The values to exclude. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of filtered values. - * @example - * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; - * - * _.differenceWith(objects, [{ 'x': 1, 'y': 2 }], _.isEqual); - * // => [{ 'x': 2, 'y': 1 }] - */ - var differenceWith = baseRest(function(array, values) { - var comparator = last(values); - if (isArrayLikeObject(comparator)) { - comparator = undefined; - } - return isArrayLikeObject(array) - ? baseDifference(array, baseFlatten(values, 1, isArrayLikeObject, true), undefined, comparator) - : []; - }); - - /** - * Creates a slice of `array` with `n` elements dropped from the beginning. - * - * @static - * @memberOf _ - * @since 0.5.0 - * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to drop. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. - * @example - * - * _.drop([1, 2, 3]); - * // => [2, 3] - * - * _.drop([1, 2, 3], 2); - * // => [3] - * - * _.drop([1, 2, 3], 5); - * // => [] - * - * _.drop([1, 2, 3], 0); - * // => [1, 2, 3] - */ - function drop(array, n, guard) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - n = (guard || n === undefined) ? 1 : toInteger(n); - return baseSlice(array, n < 0 ? 0 : n, length); - } - - /** - * Creates a slice of `array` with `n` elements dropped from the end. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to drop. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. - * @example - * - * _.dropRight([1, 2, 3]); - * // => [1, 2] - * - * _.dropRight([1, 2, 3], 2); - * // => [1] - * - * _.dropRight([1, 2, 3], 5); - * // => [] - * - * _.dropRight([1, 2, 3], 0); - * // => [1, 2, 3] - */ - function dropRight(array, n, guard) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - n = (guard || n === undefined) ? 1 : toInteger(n); - n = length - n; - return baseSlice(array, 0, n < 0 ? 0 : n); - } - - /** - * Creates a slice of `array` excluding elements dropped from the end. - * Elements are dropped until `predicate` returns falsey. The predicate is - * invoked with three arguments: (value, index, array). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. - * @example - * - * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': false } - * ]; - * - * _.dropRightWhile(users, function(o) { return !o.active; }); - * // => objects for ['barney'] - * - * // The `_.matches` iteratee shorthand. - * _.dropRightWhile(users, { 'user': 'pebbles', 'active': false }); - * // => objects for ['barney', 'fred'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.dropRightWhile(users, ['active', false]); - * // => objects for ['barney'] - * - * // The `_.property` iteratee shorthand. - * _.dropRightWhile(users, 'active'); - * // => objects for ['barney', 'fred', 'pebbles'] - */ - function dropRightWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3), true, true) - : []; - } - - /** - * Creates a slice of `array` excluding elements dropped from the beginning. - * Elements are dropped until `predicate` returns falsey. The predicate is - * invoked with three arguments: (value, index, array). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. - * @example - * - * var users = [ - * { 'user': 'barney', 'active': false }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': true } - * ]; - * - * _.dropWhile(users, function(o) { return !o.active; }); - * // => objects for ['pebbles'] - * - * // The `_.matches` iteratee shorthand. - * _.dropWhile(users, { 'user': 'barney', 'active': false }); - * // => objects for ['fred', 'pebbles'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.dropWhile(users, ['active', false]); - * // => objects for ['pebbles'] - * - * // The `_.property` iteratee shorthand. - * _.dropWhile(users, 'active'); - * // => objects for ['barney', 'fred', 'pebbles'] - */ - function dropWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3), true) - : []; - } - - /** - * Fills elements of `array` with `value` from `start` up to, but not - * including, `end`. - * - * **Note:** This method mutates `array`. - * - * @static - * @memberOf _ - * @since 3.2.0 - * @category Array - * @param {Array} array The array to fill. - * @param {*} value The value to fill `array` with. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns `array`. - * @example - * - * var array = [1, 2, 3]; - * - * _.fill(array, 'a'); - * console.log(array); - * // => ['a', 'a', 'a'] - * - * _.fill(Array(3), 2); - * // => [2, 2, 2] - * - * _.fill([4, 6, 8, 10], '*', 1, 3); - * // => [4, '*', '*', 10] - */ - function fill(array, value, start, end) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - if (start && typeof start != 'number' && isIterateeCall(array, value, start)) { - start = 0; - end = length; - } - return baseFill(array, value, start, end); - } - - /** - * This method is like `_.find` except that it returns the index of the first - * element `predicate` returns truthy for instead of the element itself. - * - * @static - * @memberOf _ - * @since 1.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=0] The index to search from. - * @returns {number} Returns the index of the found element, else `-1`. - * @example - * - * var users = [ - * { 'user': 'barney', 'active': false }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': true } - * ]; - * - * _.findIndex(users, function(o) { return o.user == 'barney'; }); - * // => 0 - * - * // The `_.matches` iteratee shorthand. - * _.findIndex(users, { 'user': 'fred', 'active': false }); - * // => 1 - * - * // The `_.matchesProperty` iteratee shorthand. - * _.findIndex(users, ['active', false]); - * // => 0 - * - * // The `_.property` iteratee shorthand. - * _.findIndex(users, 'active'); - * // => 2 - */ - function findIndex(array, predicate, fromIndex) { - var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = fromIndex == null ? 0 : toInteger(fromIndex); - if (index < 0) { - index = nativeMax(length + index, 0); - } - return baseFindIndex(array, getIteratee(predicate, 3), index); - } - - /** - * This method is like `_.findIndex` except that it iterates over elements - * of `collection` from right to left. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=array.length-1] The index to search from. - * @returns {number} Returns the index of the found element, else `-1`. - * @example - * - * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': false } - * ]; - * - * _.findLastIndex(users, function(o) { return o.user == 'pebbles'; }); - * // => 2 - * - * // The `_.matches` iteratee shorthand. - * _.findLastIndex(users, { 'user': 'barney', 'active': true }); - * // => 0 - * - * // The `_.matchesProperty` iteratee shorthand. - * _.findLastIndex(users, ['active', false]); - * // => 2 - * - * // The `_.property` iteratee shorthand. - * _.findLastIndex(users, 'active'); - * // => 0 - */ - function findLastIndex(array, predicate, fromIndex) { - var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = length - 1; - if (fromIndex !== undefined) { - index = toInteger(fromIndex); - index = fromIndex < 0 - ? nativeMax(length + index, 0) - : nativeMin(index, length - 1); - } - return baseFindIndex(array, getIteratee(predicate, 3), index, true); - } - - /** - * Flattens `array` a single level deep. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to flatten. - * @returns {Array} Returns the new flattened array. - * @example - * - * _.flatten([1, [2, [3, [4]], 5]]); - * // => [1, 2, [3, [4]], 5] - */ - function flatten(array) { - var length = array == null ? 0 : array.length; - return length ? baseFlatten(array, 1) : []; - } - - /** - * Recursively flattens `array`. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to flatten. - * @returns {Array} Returns the new flattened array. - * @example - * - * _.flattenDeep([1, [2, [3, [4]], 5]]); - * // => [1, 2, 3, 4, 5] - */ - function flattenDeep(array) { - var length = array == null ? 0 : array.length; - return length ? baseFlatten(array, INFINITY) : []; - } - - /** - * Recursively flatten `array` up to `depth` times. - * - * @static - * @memberOf _ - * @since 4.4.0 - * @category Array - * @param {Array} array The array to flatten. - * @param {number} [depth=1] The maximum recursion depth. - * @returns {Array} Returns the new flattened array. - * @example - * - * var array = [1, [2, [3, [4]], 5]]; - * - * _.flattenDepth(array, 1); - * // => [1, 2, [3, [4]], 5] - * - * _.flattenDepth(array, 2); - * // => [1, 2, 3, [4], 5] - */ - function flattenDepth(array, depth) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - depth = depth === undefined ? 1 : toInteger(depth); - return baseFlatten(array, depth); - } - - /** - * The inverse of `_.toPairs`; this method returns an object composed - * from key-value `pairs`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} pairs The key-value pairs. - * @returns {Object} Returns the new object. - * @example - * - * _.fromPairs([['a', 1], ['b', 2]]); - * // => { 'a': 1, 'b': 2 } - */ - function fromPairs(pairs) { - var index = -1, - length = pairs == null ? 0 : pairs.length, - result = {}; - - while (++index < length) { - var pair = pairs[index]; - result[pair[0]] = pair[1]; - } - return result; - } - - /** - * Gets the first element of `array`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @alias first - * @category Array - * @param {Array} array The array to query. - * @returns {*} Returns the first element of `array`. - * @example - * - * _.head([1, 2, 3]); - * // => 1 - * - * _.head([]); - * // => undefined - */ - function head(array) { - return (array && array.length) ? array[0] : undefined; - } - - /** - * Gets the index at which the first occurrence of `value` is found in `array` - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. If `fromIndex` is negative, it's used as the - * offset from the end of `array`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} [fromIndex=0] The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. - * @example - * - * _.indexOf([1, 2, 1, 2], 2); - * // => 1 - * - * // Search from the `fromIndex`. - * _.indexOf([1, 2, 1, 2], 2, 2); - * // => 3 - */ - function indexOf(array, value, fromIndex) { - var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = fromIndex == null ? 0 : toInteger(fromIndex); - if (index < 0) { - index = nativeMax(length + index, 0); - } - return baseIndexOf(array, value, index); - } - - /** - * Gets all but the last element of `array`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to query. - * @returns {Array} Returns the slice of `array`. - * @example - * - * _.initial([1, 2, 3]); - * // => [1, 2] - */ - function initial(array) { - var length = array == null ? 0 : array.length; - return length ? baseSlice(array, 0, -1) : []; - } - - /** - * Creates an array of unique values that are included in all given arrays - * using [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. The order and references of result values are - * determined by the first array. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @returns {Array} Returns the new array of intersecting values. - * @example - * - * _.intersection([2, 1], [2, 3]); - * // => [2] - */ - var intersection = baseRest(function(arrays) { - var mapped = arrayMap(arrays, castArrayLikeObject); - return (mapped.length && mapped[0] === arrays[0]) - ? baseIntersection(mapped) - : []; - }); - - /** - * This method is like `_.intersection` except that it accepts `iteratee` - * which is invoked for each element of each `arrays` to generate the criterion - * by which they're compared. The order and references of result values are - * determined by the first array. The iteratee is invoked with one argument: - * (value). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of intersecting values. - * @example - * - * _.intersectionBy([2.1, 1.2], [2.3, 3.4], Math.floor); - * // => [2.1] - * - * // The `_.property` iteratee shorthand. - * _.intersectionBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 1 }] - */ - var intersectionBy = baseRest(function(arrays) { - var iteratee = last(arrays), - mapped = arrayMap(arrays, castArrayLikeObject); - - if (iteratee === last(mapped)) { - iteratee = undefined; - } else { - mapped.pop(); - } - return (mapped.length && mapped[0] === arrays[0]) - ? baseIntersection(mapped, getIteratee(iteratee, 2)) - : []; - }); - - /** - * This method is like `_.intersection` except that it accepts `comparator` - * which is invoked to compare elements of `arrays`. The order and references - * of result values are determined by the first array. The comparator is - * invoked with two arguments: (arrVal, othVal). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of intersecting values. - * @example - * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; - * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; - * - * _.intersectionWith(objects, others, _.isEqual); - * // => [{ 'x': 1, 'y': 2 }] - */ - var intersectionWith = baseRest(function(arrays) { - var comparator = last(arrays), - mapped = arrayMap(arrays, castArrayLikeObject); - - comparator = typeof comparator == 'function' ? comparator : undefined; - if (comparator) { - mapped.pop(); - } - return (mapped.length && mapped[0] === arrays[0]) - ? baseIntersection(mapped, undefined, comparator) - : []; - }); - - /** - * Converts all elements in `array` into a string separated by `separator`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to convert. - * @param {string} [separator=','] The element separator. - * @returns {string} Returns the joined string. - * @example - * - * _.join(['a', 'b', 'c'], '~'); - * // => 'a~b~c' - */ - function join(array, separator) { - return array == null ? '' : nativeJoin.call(array, separator); - } - - /** - * Gets the last element of `array`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to query. - * @returns {*} Returns the last element of `array`. - * @example - * - * _.last([1, 2, 3]); - * // => 3 - */ - function last(array) { - var length = array == null ? 0 : array.length; - return length ? array[length - 1] : undefined; - } - - /** - * This method is like `_.indexOf` except that it iterates over elements of - * `array` from right to left. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @param {number} [fromIndex=array.length-1] The index to search from. - * @returns {number} Returns the index of the matched value, else `-1`. - * @example - * - * _.lastIndexOf([1, 2, 1, 2], 2); - * // => 3 - * - * // Search from the `fromIndex`. - * _.lastIndexOf([1, 2, 1, 2], 2, 2); - * // => 1 - */ - function lastIndexOf(array, value, fromIndex) { - var length = array == null ? 0 : array.length; - if (!length) { - return -1; - } - var index = length; - if (fromIndex !== undefined) { - index = toInteger(fromIndex); - index = index < 0 ? nativeMax(length + index, 0) : nativeMin(index, length - 1); - } - return value === value - ? strictLastIndexOf(array, value, index) - : baseFindIndex(array, baseIsNaN, index, true); - } - - /** - * Gets the element at index `n` of `array`. If `n` is negative, the nth - * element from the end is returned. - * - * @static - * @memberOf _ - * @since 4.11.0 - * @category Array - * @param {Array} array The array to query. - * @param {number} [n=0] The index of the element to return. - * @returns {*} Returns the nth element of `array`. - * @example - * - * var array = ['a', 'b', 'c', 'd']; - * - * _.nth(array, 1); - * // => 'b' - * - * _.nth(array, -2); - * // => 'c'; - */ - function nth(array, n) { - return (array && array.length) ? baseNth(array, toInteger(n)) : undefined; - } - - /** - * Removes all given values from `array` using - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. - * - * **Note:** Unlike `_.without`, this method mutates `array`. Use `_.remove` - * to remove elements from an array by predicate. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Array - * @param {Array} array The array to modify. - * @param {...*} [values] The values to remove. - * @returns {Array} Returns `array`. - * @example - * - * var array = ['a', 'b', 'c', 'a', 'b', 'c']; - * - * _.pull(array, 'a', 'c'); - * console.log(array); - * // => ['b', 'b'] - */ - var pull = baseRest(pullAll); - - /** - * This method is like `_.pull` except that it accepts an array of values to remove. - * - * **Note:** Unlike `_.difference`, this method mutates `array`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. - * @returns {Array} Returns `array`. - * @example - * - * var array = ['a', 'b', 'c', 'a', 'b', 'c']; - * - * _.pullAll(array, ['a', 'c']); - * console.log(array); - * // => ['b', 'b'] - */ - function pullAll(array, values) { - return (array && array.length && values && values.length) - ? basePullAll(array, values) - : array; - } - - /** - * This method is like `_.pullAll` except that it accepts `iteratee` which is - * invoked for each element of `array` and `values` to generate the criterion - * by which they're compared. The iteratee is invoked with one argument: (value). - * - * **Note:** Unlike `_.differenceBy`, this method mutates `array`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns `array`. - * @example - * - * var array = [{ 'x': 1 }, { 'x': 2 }, { 'x': 3 }, { 'x': 1 }]; - * - * _.pullAllBy(array, [{ 'x': 1 }, { 'x': 3 }], 'x'); - * console.log(array); - * // => [{ 'x': 2 }] - */ - function pullAllBy(array, values, iteratee) { - return (array && array.length && values && values.length) - ? basePullAll(array, values, getIteratee(iteratee, 2)) - : array; - } - - /** - * This method is like `_.pullAll` except that it accepts `comparator` which - * is invoked to compare elements of `array` to `values`. The comparator is - * invoked with two arguments: (arrVal, othVal). - * - * **Note:** Unlike `_.differenceWith`, this method mutates `array`. - * - * @static - * @memberOf _ - * @since 4.6.0 - * @category Array - * @param {Array} array The array to modify. - * @param {Array} values The values to remove. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns `array`. - * @example - * - * var array = [{ 'x': 1, 'y': 2 }, { 'x': 3, 'y': 4 }, { 'x': 5, 'y': 6 }]; - * - * _.pullAllWith(array, [{ 'x': 3, 'y': 4 }], _.isEqual); - * console.log(array); - * // => [{ 'x': 1, 'y': 2 }, { 'x': 5, 'y': 6 }] - */ - function pullAllWith(array, values, comparator) { - return (array && array.length && values && values.length) - ? basePullAll(array, values, undefined, comparator) - : array; - } - - /** - * Removes elements from `array` corresponding to `indexes` and returns an - * array of removed elements. - * - * **Note:** Unlike `_.at`, this method mutates `array`. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to modify. - * @param {...(number|number[])} [indexes] The indexes of elements to remove. - * @returns {Array} Returns the new array of removed elements. - * @example - * - * var array = ['a', 'b', 'c', 'd']; - * var pulled = _.pullAt(array, [1, 3]); - * - * console.log(array); - * // => ['a', 'c'] - * - * console.log(pulled); - * // => ['b', 'd'] - */ - var pullAt = flatRest(function(array, indexes) { - var length = array == null ? 0 : array.length, - result = baseAt(array, indexes); - - basePullAt(array, arrayMap(indexes, function(index) { - return isIndex(index, length) ? +index : index; - }).sort(compareAscending)); - - return result; - }); - - /** - * Removes all elements from `array` that `predicate` returns truthy for - * and returns an array of the removed elements. The predicate is invoked - * with three arguments: (value, index, array). - * - * **Note:** Unlike `_.filter`, this method mutates `array`. Use `_.pull` - * to pull elements from an array by value. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Array - * @param {Array} array The array to modify. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new array of removed elements. - * @example - * - * var array = [1, 2, 3, 4]; - * var evens = _.remove(array, function(n) { - * return n % 2 == 0; - * }); - * - * console.log(array); - * // => [1, 3] - * - * console.log(evens); - * // => [2, 4] - */ - function remove(array, predicate) { - var result = []; - if (!(array && array.length)) { - return result; - } - var index = -1, - indexes = [], - length = array.length; - - predicate = getIteratee(predicate, 3); - while (++index < length) { - var value = array[index]; - if (predicate(value, index, array)) { - result.push(value); - indexes.push(index); - } - } - basePullAt(array, indexes); - return result; - } - - /** - * Reverses `array` so that the first element becomes the last, the second - * element becomes the second to last, and so on. - * - * **Note:** This method mutates `array` and is based on - * [`Array#reverse`](https://mdn.io/Array/reverse). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to modify. - * @returns {Array} Returns `array`. - * @example - * - * var array = [1, 2, 3]; - * - * _.reverse(array); - * // => [3, 2, 1] - * - * console.log(array); - * // => [3, 2, 1] - */ - function reverse(array) { - return array == null ? array : nativeReverse.call(array); - } - - /** - * Creates a slice of `array` from `start` up to, but not including, `end`. - * - * **Note:** This method is used instead of - * [`Array#slice`](https://mdn.io/Array/slice) to ensure dense arrays are - * returned. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to slice. - * @param {number} [start=0] The start position. - * @param {number} [end=array.length] The end position. - * @returns {Array} Returns the slice of `array`. - */ - function slice(array, start, end) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - if (end && typeof end != 'number' && isIterateeCall(array, start, end)) { - start = 0; - end = length; - } - else { - start = start == null ? 0 : toInteger(start); - end = end === undefined ? length : toInteger(end); - } - return baseSlice(array, start, end); - } - - /** - * Uses a binary search to determine the lowest index at which `value` - * should be inserted into `array` in order to maintain its sort order. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. - * @example - * - * _.sortedIndex([30, 50], 40); - * // => 1 - */ - function sortedIndex(array, value) { - return baseSortedIndex(array, value); - } - - /** - * This method is like `_.sortedIndex` except that it accepts `iteratee` - * which is invoked for `value` and each element of `array` to compute their - * sort ranking. The iteratee is invoked with one argument: (value). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. - * @example - * - * var objects = [{ 'x': 4 }, { 'x': 5 }]; - * - * _.sortedIndexBy(objects, { 'x': 4 }, function(o) { return o.x; }); - * // => 0 - * - * // The `_.property` iteratee shorthand. - * _.sortedIndexBy(objects, { 'x': 4 }, 'x'); - * // => 0 - */ - function sortedIndexBy(array, value, iteratee) { - return baseSortedIndexBy(array, value, getIteratee(iteratee, 2)); - } - - /** - * This method is like `_.indexOf` except that it performs a binary - * search on a sorted `array`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @returns {number} Returns the index of the matched value, else `-1`. - * @example - * - * _.sortedIndexOf([4, 5, 5, 5, 6], 5); - * // => 1 - */ - function sortedIndexOf(array, value) { - var length = array == null ? 0 : array.length; - if (length) { - var index = baseSortedIndex(array, value); - if (index < length && eq(array[index], value)) { - return index; - } - } - return -1; - } - - /** - * This method is like `_.sortedIndex` except that it returns the highest - * index at which `value` should be inserted into `array` in order to - * maintain its sort order. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. - * @example - * - * _.sortedLastIndex([4, 5, 5, 5, 6], 5); - * // => 4 - */ - function sortedLastIndex(array, value) { - return baseSortedIndex(array, value, true); - } - - /** - * This method is like `_.sortedLastIndex` except that it accepts `iteratee` - * which is invoked for `value` and each element of `array` to compute their - * sort ranking. The iteratee is invoked with one argument: (value). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The sorted array to inspect. - * @param {*} value The value to evaluate. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {number} Returns the index at which `value` should be inserted - * into `array`. - * @example - * - * var objects = [{ 'x': 4 }, { 'x': 5 }]; - * - * _.sortedLastIndexBy(objects, { 'x': 4 }, function(o) { return o.x; }); - * // => 1 - * - * // The `_.property` iteratee shorthand. - * _.sortedLastIndexBy(objects, { 'x': 4 }, 'x'); - * // => 1 - */ - function sortedLastIndexBy(array, value, iteratee) { - return baseSortedIndexBy(array, value, getIteratee(iteratee, 2), true); - } - - /** - * This method is like `_.lastIndexOf` except that it performs a binary - * search on a sorted `array`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {*} value The value to search for. - * @returns {number} Returns the index of the matched value, else `-1`. - * @example - * - * _.sortedLastIndexOf([4, 5, 5, 5, 6], 5); - * // => 3 - */ - function sortedLastIndexOf(array, value) { - var length = array == null ? 0 : array.length; - if (length) { - var index = baseSortedIndex(array, value, true) - 1; - if (eq(array[index], value)) { - return index; - } - } - return -1; - } - - /** - * This method is like `_.uniq` except that it's designed and optimized - * for sorted arrays. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @returns {Array} Returns the new duplicate free array. - * @example - * - * _.sortedUniq([1, 1, 2]); - * // => [1, 2] - */ - function sortedUniq(array) { - return (array && array.length) - ? baseSortedUniq(array) - : []; - } - - /** - * This method is like `_.uniqBy` except that it's designed and optimized - * for sorted arrays. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [iteratee] The iteratee invoked per element. - * @returns {Array} Returns the new duplicate free array. - * @example - * - * _.sortedUniqBy([1.1, 1.2, 2.3, 2.4], Math.floor); - * // => [1.1, 2.3] - */ - function sortedUniqBy(array, iteratee) { - return (array && array.length) - ? baseSortedUniq(array, getIteratee(iteratee, 2)) - : []; - } - - /** - * Gets all but the first element of `array`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to query. - * @returns {Array} Returns the slice of `array`. - * @example - * - * _.tail([1, 2, 3]); - * // => [2, 3] - */ - function tail(array) { - var length = array == null ? 0 : array.length; - return length ? baseSlice(array, 1, length) : []; - } - - /** - * Creates a slice of `array` with `n` elements taken from the beginning. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to take. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. - * @example - * - * _.take([1, 2, 3]); - * // => [1] - * - * _.take([1, 2, 3], 2); - * // => [1, 2] - * - * _.take([1, 2, 3], 5); - * // => [1, 2, 3] - * - * _.take([1, 2, 3], 0); - * // => [] - */ - function take(array, n, guard) { - if (!(array && array.length)) { - return []; - } - n = (guard || n === undefined) ? 1 : toInteger(n); - return baseSlice(array, 0, n < 0 ? 0 : n); - } - - /** - * Creates a slice of `array` with `n` elements taken from the end. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. - * @param {number} [n=1] The number of elements to take. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the slice of `array`. - * @example - * - * _.takeRight([1, 2, 3]); - * // => [3] - * - * _.takeRight([1, 2, 3], 2); - * // => [2, 3] - * - * _.takeRight([1, 2, 3], 5); - * // => [1, 2, 3] - * - * _.takeRight([1, 2, 3], 0); - * // => [] - */ - function takeRight(array, n, guard) { - var length = array == null ? 0 : array.length; - if (!length) { - return []; - } - n = (guard || n === undefined) ? 1 : toInteger(n); - n = length - n; - return baseSlice(array, n < 0 ? 0 : n, length); - } - - /** - * Creates a slice of `array` with elements taken from the end. Elements are - * taken until `predicate` returns falsey. The predicate is invoked with - * three arguments: (value, index, array). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. - * @example - * - * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': false } - * ]; - * - * _.takeRightWhile(users, function(o) { return !o.active; }); - * // => objects for ['fred', 'pebbles'] - * - * // The `_.matches` iteratee shorthand. - * _.takeRightWhile(users, { 'user': 'pebbles', 'active': false }); - * // => objects for ['pebbles'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.takeRightWhile(users, ['active', false]); - * // => objects for ['fred', 'pebbles'] - * - * // The `_.property` iteratee shorthand. - * _.takeRightWhile(users, 'active'); - * // => [] - */ - function takeRightWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3), false, true) - : []; - } - - /** - * Creates a slice of `array` with elements taken from the beginning. Elements - * are taken until `predicate` returns falsey. The predicate is invoked with - * three arguments: (value, index, array). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Array - * @param {Array} array The array to query. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the slice of `array`. - * @example - * - * var users = [ - * { 'user': 'barney', 'active': false }, - * { 'user': 'fred', 'active': false }, - * { 'user': 'pebbles', 'active': true } - * ]; - * - * _.takeWhile(users, function(o) { return !o.active; }); - * // => objects for ['barney', 'fred'] - * - * // The `_.matches` iteratee shorthand. - * _.takeWhile(users, { 'user': 'barney', 'active': false }); - * // => objects for ['barney'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.takeWhile(users, ['active', false]); - * // => objects for ['barney', 'fred'] - * - * // The `_.property` iteratee shorthand. - * _.takeWhile(users, 'active'); - * // => [] - */ - function takeWhile(array, predicate) { - return (array && array.length) - ? baseWhile(array, getIteratee(predicate, 3)) - : []; - } - - /** - * Creates an array of unique values, in order, from all given arrays using - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @returns {Array} Returns the new array of combined values. - * @example - * - * _.union([2], [1, 2]); - * // => [2, 1] - */ - var union = baseRest(function(arrays) { - return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true)); - }); - - /** - * This method is like `_.union` except that it accepts `iteratee` which is - * invoked for each element of each `arrays` to generate the criterion by - * which uniqueness is computed. Result values are chosen from the first - * array in which the value occurs. The iteratee is invoked with one argument: - * (value). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of combined values. - * @example - * - * _.unionBy([2.1], [1.2, 2.3], Math.floor); - * // => [2.1, 1.2] - * - * // The `_.property` iteratee shorthand. - * _.unionBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 1 }, { 'x': 2 }] - */ - var unionBy = baseRest(function(arrays) { - var iteratee = last(arrays); - if (isArrayLikeObject(iteratee)) { - iteratee = undefined; - } - return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true), getIteratee(iteratee, 2)); - }); - - /** - * This method is like `_.union` except that it accepts `comparator` which - * is invoked to compare elements of `arrays`. Result values are chosen from - * the first array in which the value occurs. The comparator is invoked - * with two arguments: (arrVal, othVal). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of combined values. - * @example - * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; - * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; - * - * _.unionWith(objects, others, _.isEqual); - * // => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 1 }] - */ - var unionWith = baseRest(function(arrays) { - var comparator = last(arrays); - comparator = typeof comparator == 'function' ? comparator : undefined; - return baseUniq(baseFlatten(arrays, 1, isArrayLikeObject, true), undefined, comparator); - }); - - /** - * Creates a duplicate-free version of an array, using - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons, in which only the first occurrence of each element - * is kept. The order of result values is determined by the order they occur - * in the array. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @returns {Array} Returns the new duplicate free array. - * @example - * - * _.uniq([2, 1, 2]); - * // => [2, 1] - */ - function uniq(array) { - return (array && array.length) ? baseUniq(array) : []; - } - - /** - * This method is like `_.uniq` except that it accepts `iteratee` which is - * invoked for each element in `array` to generate the criterion by which - * uniqueness is computed. The order of result values is determined by the - * order they occur in the array. The iteratee is invoked with one argument: - * (value). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new duplicate free array. - * @example - * - * _.uniqBy([2.1, 1.2, 2.3], Math.floor); - * // => [2.1, 1.2] - * - * // The `_.property` iteratee shorthand. - * _.uniqBy([{ 'x': 1 }, { 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 1 }, { 'x': 2 }] - */ - function uniqBy(array, iteratee) { - return (array && array.length) ? baseUniq(array, getIteratee(iteratee, 2)) : []; - } - - /** - * This method is like `_.uniq` except that it accepts `comparator` which - * is invoked to compare elements of `array`. The order of result values is - * determined by the order they occur in the array.The comparator is invoked - * with two arguments: (arrVal, othVal). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new duplicate free array. - * @example - * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }, { 'x': 1, 'y': 2 }]; - * - * _.uniqWith(objects, _.isEqual); - * // => [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }] - */ - function uniqWith(array, comparator) { - comparator = typeof comparator == 'function' ? comparator : undefined; - return (array && array.length) ? baseUniq(array, undefined, comparator) : []; - } - - /** - * This method is like `_.zip` except that it accepts an array of grouped - * elements and creates an array regrouping the elements to their pre-zip - * configuration. - * - * @static - * @memberOf _ - * @since 1.2.0 - * @category Array - * @param {Array} array The array of grouped elements to process. - * @returns {Array} Returns the new array of regrouped elements. - * @example - * - * var zipped = _.zip(['a', 'b'], [1, 2], [true, false]); - * // => [['a', 1, true], ['b', 2, false]] - * - * _.unzip(zipped); - * // => [['a', 'b'], [1, 2], [true, false]] - */ - function unzip(array) { - if (!(array && array.length)) { - return []; - } - var length = 0; - array = arrayFilter(array, function(group) { - if (isArrayLikeObject(group)) { - length = nativeMax(group.length, length); - return true; - } - }); - return baseTimes(length, function(index) { - return arrayMap(array, baseProperty(index)); - }); - } - - /** - * This method is like `_.unzip` except that it accepts `iteratee` to specify - * how regrouped values should be combined. The iteratee is invoked with the - * elements of each group: (...group). - * - * @static - * @memberOf _ - * @since 3.8.0 - * @category Array - * @param {Array} array The array of grouped elements to process. - * @param {Function} [iteratee=_.identity] The function to combine - * regrouped values. - * @returns {Array} Returns the new array of regrouped elements. - * @example - * - * var zipped = _.zip([1, 2], [10, 20], [100, 200]); - * // => [[1, 10, 100], [2, 20, 200]] - * - * _.unzipWith(zipped, _.add); - * // => [3, 30, 300] - */ - function unzipWith(array, iteratee) { - if (!(array && array.length)) { - return []; - } - var result = unzip(array); - if (iteratee == null) { - return result; - } - return arrayMap(result, function(group) { - return apply(iteratee, undefined, group); - }); - } - - /** - * Creates an array excluding all given values using - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * for equality comparisons. - * - * **Note:** Unlike `_.pull`, this method returns a new array. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {Array} array The array to inspect. - * @param {...*} [values] The values to exclude. - * @returns {Array} Returns the new array of filtered values. - * @see _.difference, _.xor - * @example - * - * _.without([2, 1, 2, 3], 1, 2); - * // => [3] - */ - var without = baseRest(function(array, values) { - return isArrayLikeObject(array) - ? baseDifference(array, values) - : []; - }); - - /** - * Creates an array of unique values that is the - * [symmetric difference](https://en.wikipedia.org/wiki/Symmetric_difference) - * of the given arrays. The order of result values is determined by the order - * they occur in the arrays. - * - * @static - * @memberOf _ - * @since 2.4.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @returns {Array} Returns the new array of filtered values. - * @see _.difference, _.without - * @example - * - * _.xor([2, 1], [2, 3]); - * // => [1, 3] - */ - var xor = baseRest(function(arrays) { - return baseXor(arrayFilter(arrays, isArrayLikeObject)); - }); - - /** - * This method is like `_.xor` except that it accepts `iteratee` which is - * invoked for each element of each `arrays` to generate the criterion by - * which by which they're compared. The order of result values is determined - * by the order they occur in the arrays. The iteratee is invoked with one - * argument: (value). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Array} Returns the new array of filtered values. - * @example - * - * _.xorBy([2.1, 1.2], [2.3, 3.4], Math.floor); - * // => [1.2, 3.4] - * - * // The `_.property` iteratee shorthand. - * _.xorBy([{ 'x': 1 }], [{ 'x': 2 }, { 'x': 1 }], 'x'); - * // => [{ 'x': 2 }] - */ - var xorBy = baseRest(function(arrays) { - var iteratee = last(arrays); - if (isArrayLikeObject(iteratee)) { - iteratee = undefined; - } - return baseXor(arrayFilter(arrays, isArrayLikeObject), getIteratee(iteratee, 2)); - }); - - /** - * This method is like `_.xor` except that it accepts `comparator` which is - * invoked to compare elements of `arrays`. The order of result values is - * determined by the order they occur in the arrays. The comparator is invoked - * with two arguments: (arrVal, othVal). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Array - * @param {...Array} [arrays] The arrays to inspect. - * @param {Function} [comparator] The comparator invoked per element. - * @returns {Array} Returns the new array of filtered values. - * @example - * - * var objects = [{ 'x': 1, 'y': 2 }, { 'x': 2, 'y': 1 }]; - * var others = [{ 'x': 1, 'y': 1 }, { 'x': 1, 'y': 2 }]; - * - * _.xorWith(objects, others, _.isEqual); - * // => [{ 'x': 2, 'y': 1 }, { 'x': 1, 'y': 1 }] - */ - var xorWith = baseRest(function(arrays) { - var comparator = last(arrays); - comparator = typeof comparator == 'function' ? comparator : undefined; - return baseXor(arrayFilter(arrays, isArrayLikeObject), undefined, comparator); - }); - - /** - * Creates an array of grouped elements, the first of which contains the - * first elements of the given arrays, the second of which contains the - * second elements of the given arrays, and so on. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Array - * @param {...Array} [arrays] The arrays to process. - * @returns {Array} Returns the new array of grouped elements. - * @example - * - * _.zip(['a', 'b'], [1, 2], [true, false]); - * // => [['a', 1, true], ['b', 2, false]] - */ - var zip = baseRest(unzip); - - /** - * This method is like `_.fromPairs` except that it accepts two arrays, - * one of property identifiers and one of corresponding values. - * - * @static - * @memberOf _ - * @since 0.4.0 - * @category Array - * @param {Array} [props=[]] The property identifiers. - * @param {Array} [values=[]] The property values. - * @returns {Object} Returns the new object. - * @example - * - * _.zipObject(['a', 'b'], [1, 2]); - * // => { 'a': 1, 'b': 2 } - */ - function zipObject(props, values) { - return baseZipObject(props || [], values || [], assignValue); - } - - /** - * This method is like `_.zipObject` except that it supports property paths. - * - * @static - * @memberOf _ - * @since 4.1.0 - * @category Array - * @param {Array} [props=[]] The property identifiers. - * @param {Array} [values=[]] The property values. - * @returns {Object} Returns the new object. - * @example - * - * _.zipObjectDeep(['a.b[0].c', 'a.b[1].d'], [1, 2]); - * // => { 'a': { 'b': [{ 'c': 1 }, { 'd': 2 }] } } - */ - function zipObjectDeep(props, values) { - return baseZipObject(props || [], values || [], baseSet); - } - - /** - * This method is like `_.zip` except that it accepts `iteratee` to specify - * how grouped values should be combined. The iteratee is invoked with the - * elements of each group: (...group). - * - * @static - * @memberOf _ - * @since 3.8.0 - * @category Array - * @param {...Array} [arrays] The arrays to process. - * @param {Function} [iteratee=_.identity] The function to combine - * grouped values. - * @returns {Array} Returns the new array of grouped elements. - * @example - * - * _.zipWith([1, 2], [10, 20], [100, 200], function(a, b, c) { - * return a + b + c; - * }); - * // => [111, 222] - */ - var zipWith = baseRest(function(arrays) { - var length = arrays.length, - iteratee = length > 1 ? arrays[length - 1] : undefined; - - iteratee = typeof iteratee == 'function' ? (arrays.pop(), iteratee) : undefined; - return unzipWith(arrays, iteratee); - }); - - /*------------------------------------------------------------------------*/ - - /** - * Creates a `lodash` wrapper instance that wraps `value` with explicit method - * chain sequences enabled. The result of such sequences must be unwrapped - * with `_#value`. - * - * @static - * @memberOf _ - * @since 1.3.0 - * @category Seq - * @param {*} value The value to wrap. - * @returns {Object} Returns the new `lodash` wrapper instance. - * @example - * - * var users = [ - * { 'user': 'barney', 'age': 36 }, - * { 'user': 'fred', 'age': 40 }, - * { 'user': 'pebbles', 'age': 1 } - * ]; - * - * var youngest = _ - * .chain(users) - * .sortBy('age') - * .map(function(o) { - * return o.user + ' is ' + o.age; - * }) - * .head() - * .value(); - * // => 'pebbles is 1' - */ - function chain(value) { - var result = lodash(value); - result.__chain__ = true; - return result; - } - - /** - * This method invokes `interceptor` and returns `value`. The interceptor - * is invoked with one argument; (value). The purpose of this method is to - * "tap into" a method chain sequence in order to modify intermediate results. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Seq - * @param {*} value The value to provide to `interceptor`. - * @param {Function} interceptor The function to invoke. - * @returns {*} Returns `value`. - * @example - * - * _([1, 2, 3]) - * .tap(function(array) { - * // Mutate input array. - * array.pop(); - * }) - * .reverse() - * .value(); - * // => [2, 1] - */ - function tap(value, interceptor) { - interceptor(value); - return value; - } - - /** - * This method is like `_.tap` except that it returns the result of `interceptor`. - * The purpose of this method is to "pass thru" values replacing intermediate - * results in a method chain sequence. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Seq - * @param {*} value The value to provide to `interceptor`. - * @param {Function} interceptor The function to invoke. - * @returns {*} Returns the result of `interceptor`. - * @example - * - * _(' abc ') - * .chain() - * .trim() - * .thru(function(value) { - * return [value]; - * }) - * .value(); - * // => ['abc'] - */ - function thru(value, interceptor) { - return interceptor(value); - } - - /** - * This method is the wrapper version of `_.at`. - * - * @name at - * @memberOf _ - * @since 1.0.0 - * @category Seq - * @param {...(string|string[])} [paths] The property paths to pick. - * @returns {Object} Returns the new `lodash` wrapper instance. - * @example - * - * var object = { 'a': [{ 'b': { 'c': 3 } }, 4] }; - * - * _(object).at(['a[0].b.c', 'a[1]']).value(); - * // => [3, 4] - */ - var wrapperAt = flatRest(function(paths) { - var length = paths.length, - start = length ? paths[0] : 0, - value = this.__wrapped__, - interceptor = function(object) { return baseAt(object, paths); }; - - if (length > 1 || this.__actions__.length || - !(value instanceof LazyWrapper) || !isIndex(start)) { - return this.thru(interceptor); - } - value = value.slice(start, +start + (length ? 1 : 0)); - value.__actions__.push({ - 'func': thru, - 'args': [interceptor], - 'thisArg': undefined - }); - return new LodashWrapper(value, this.__chain__).thru(function(array) { - if (length && !array.length) { - array.push(undefined); - } - return array; - }); - }); - - /** - * Creates a `lodash` wrapper instance with explicit method chain sequences enabled. - * - * @name chain - * @memberOf _ - * @since 0.1.0 - * @category Seq - * @returns {Object} Returns the new `lodash` wrapper instance. - * @example - * - * var users = [ - * { 'user': 'barney', 'age': 36 }, - * { 'user': 'fred', 'age': 40 } - * ]; - * - * // A sequence without explicit chaining. - * _(users).head(); - * // => { 'user': 'barney', 'age': 36 } - * - * // A sequence with explicit chaining. - * _(users) - * .chain() - * .head() - * .pick('user') - * .value(); - * // => { 'user': 'barney' } - */ - function wrapperChain() { - return chain(this); - } - - /** - * Executes the chain sequence and returns the wrapped result. - * - * @name commit - * @memberOf _ - * @since 3.2.0 - * @category Seq - * @returns {Object} Returns the new `lodash` wrapper instance. - * @example - * - * var array = [1, 2]; - * var wrapped = _(array).push(3); - * - * console.log(array); - * // => [1, 2] - * - * wrapped = wrapped.commit(); - * console.log(array); - * // => [1, 2, 3] - * - * wrapped.last(); - * // => 3 - * - * console.log(array); - * // => [1, 2, 3] - */ - function wrapperCommit() { - return new LodashWrapper(this.value(), this.__chain__); - } - - /** - * Gets the next value on a wrapped object following the - * [iterator protocol](https://mdn.io/iteration_protocols#iterator). - * - * @name next - * @memberOf _ - * @since 4.0.0 - * @category Seq - * @returns {Object} Returns the next iterator value. - * @example - * - * var wrapped = _([1, 2]); - * - * wrapped.next(); - * // => { 'done': false, 'value': 1 } - * - * wrapped.next(); - * // => { 'done': false, 'value': 2 } - * - * wrapped.next(); - * // => { 'done': true, 'value': undefined } - */ - function wrapperNext() { - if (this.__values__ === undefined) { - this.__values__ = toArray(this.value()); - } - var done = this.__index__ >= this.__values__.length, - value = done ? undefined : this.__values__[this.__index__++]; - - return { 'done': done, 'value': value }; - } - - /** - * Enables the wrapper to be iterable. - * - * @name Symbol.iterator - * @memberOf _ - * @since 4.0.0 - * @category Seq - * @returns {Object} Returns the wrapper object. - * @example - * - * var wrapped = _([1, 2]); - * - * wrapped[Symbol.iterator]() === wrapped; - * // => true - * - * Array.from(wrapped); - * // => [1, 2] - */ - function wrapperToIterator() { - return this; - } - - /** - * Creates a clone of the chain sequence planting `value` as the wrapped value. - * - * @name plant - * @memberOf _ - * @since 3.2.0 - * @category Seq - * @param {*} value The value to plant. - * @returns {Object} Returns the new `lodash` wrapper instance. - * @example - * - * function square(n) { - * return n * n; - * } - * - * var wrapped = _([1, 2]).map(square); - * var other = wrapped.plant([3, 4]); - * - * other.value(); - * // => [9, 16] - * - * wrapped.value(); - * // => [1, 4] - */ - function wrapperPlant(value) { - var result, - parent = this; - - while (parent instanceof baseLodash) { - var clone = wrapperClone(parent); - clone.__index__ = 0; - clone.__values__ = undefined; - if (result) { - previous.__wrapped__ = clone; - } else { - result = clone; - } - var previous = clone; - parent = parent.__wrapped__; - } - previous.__wrapped__ = value; - return result; - } - - /** - * This method is the wrapper version of `_.reverse`. - * - * **Note:** This method mutates the wrapped array. - * - * @name reverse - * @memberOf _ - * @since 0.1.0 - * @category Seq - * @returns {Object} Returns the new `lodash` wrapper instance. - * @example - * - * var array = [1, 2, 3]; - * - * _(array).reverse().value() - * // => [3, 2, 1] - * - * console.log(array); - * // => [3, 2, 1] - */ - function wrapperReverse() { - var value = this.__wrapped__; - if (value instanceof LazyWrapper) { - var wrapped = value; - if (this.__actions__.length) { - wrapped = new LazyWrapper(this); - } - wrapped = wrapped.reverse(); - wrapped.__actions__.push({ - 'func': thru, - 'args': [reverse], - 'thisArg': undefined - }); - return new LodashWrapper(wrapped, this.__chain__); - } - return this.thru(reverse); - } - - /** - * Executes the chain sequence to resolve the unwrapped value. - * - * @name value - * @memberOf _ - * @since 0.1.0 - * @alias toJSON, valueOf - * @category Seq - * @returns {*} Returns the resolved unwrapped value. - * @example - * - * _([1, 2, 3]).value(); - * // => [1, 2, 3] - */ - function wrapperValue() { - return baseWrapperValue(this.__wrapped__, this.__actions__); - } - - /*------------------------------------------------------------------------*/ - - /** - * Creates an object composed of keys generated from the results of running - * each element of `collection` thru `iteratee`. The corresponding value of - * each key is the number of times the key was returned by `iteratee`. The - * iteratee is invoked with one argument: (value). - * - * @static - * @memberOf _ - * @since 0.5.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The iteratee to transform keys. - * @returns {Object} Returns the composed aggregate object. - * @example - * - * _.countBy([6.1, 4.2, 6.3], Math.floor); - * // => { '4': 1, '6': 2 } - * - * // The `_.property` iteratee shorthand. - * _.countBy(['one', 'two', 'three'], 'length'); - * // => { '3': 2, '5': 1 } - */ - var countBy = createAggregator(function(result, value, key) { - if (hasOwnProperty.call(result, key)) { - ++result[key]; - } else { - baseAssignValue(result, key, 1); - } - }); - - /** - * Checks if `predicate` returns truthy for **all** elements of `collection`. - * Iteration is stopped once `predicate` returns falsey. The predicate is - * invoked with three arguments: (value, index|key, collection). - * - * **Note:** This method returns `true` for - * [empty collections](https://en.wikipedia.org/wiki/Empty_set) because - * [everything is true](https://en.wikipedia.org/wiki/Vacuous_truth) of - * elements of empty collections. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {boolean} Returns `true` if all elements pass the predicate check, - * else `false`. - * @example - * - * _.every([true, 1, null, 'yes'], Boolean); - * // => false - * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': false }, - * { 'user': 'fred', 'age': 40, 'active': false } - * ]; - * - * // The `_.matches` iteratee shorthand. - * _.every(users, { 'user': 'barney', 'active': false }); - * // => false - * - * // The `_.matchesProperty` iteratee shorthand. - * _.every(users, ['active', false]); - * // => true - * - * // The `_.property` iteratee shorthand. - * _.every(users, 'active'); - * // => false - */ - function every(collection, predicate, guard) { - var func = isArray(collection) ? arrayEvery : baseEvery; - if (guard && isIterateeCall(collection, predicate, guard)) { - predicate = undefined; - } - return func(collection, getIteratee(predicate, 3)); - } - - /** - * Iterates over elements of `collection`, returning an array of all elements - * `predicate` returns truthy for. The predicate is invoked with three - * arguments: (value, index|key, collection). - * - * **Note:** Unlike `_.remove`, this method returns a new array. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new filtered array. - * @see _.reject - * @example - * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': true }, - * { 'user': 'fred', 'age': 40, 'active': false } - * ]; - * - * _.filter(users, function(o) { return !o.active; }); - * // => objects for ['fred'] - * - * // The `_.matches` iteratee shorthand. - * _.filter(users, { 'age': 36, 'active': true }); - * // => objects for ['barney'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.filter(users, ['active', false]); - * // => objects for ['fred'] - * - * // The `_.property` iteratee shorthand. - * _.filter(users, 'active'); - * // => objects for ['barney'] - */ - function filter(collection, predicate) { - var func = isArray(collection) ? arrayFilter : baseFilter; - return func(collection, getIteratee(predicate, 3)); - } - - /** - * Iterates over elements of `collection`, returning the first element - * `predicate` returns truthy for. The predicate is invoked with three - * arguments: (value, index|key, collection). - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=0] The index to search from. - * @returns {*} Returns the matched element, else `undefined`. - * @example - * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': true }, - * { 'user': 'fred', 'age': 40, 'active': false }, - * { 'user': 'pebbles', 'age': 1, 'active': true } - * ]; - * - * _.find(users, function(o) { return o.age < 40; }); - * // => object for 'barney' - * - * // The `_.matches` iteratee shorthand. - * _.find(users, { 'age': 1, 'active': true }); - * // => object for 'pebbles' - * - * // The `_.matchesProperty` iteratee shorthand. - * _.find(users, ['active', false]); - * // => object for 'fred' - * - * // The `_.property` iteratee shorthand. - * _.find(users, 'active'); - * // => object for 'barney' - */ - var find = createFind(findIndex); - - /** - * This method is like `_.find` except that it iterates over elements of - * `collection` from right to left. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Collection - * @param {Array|Object} collection The collection to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param {number} [fromIndex=collection.length-1] The index to search from. - * @returns {*} Returns the matched element, else `undefined`. - * @example - * - * _.findLast([1, 2, 3, 4], function(n) { - * return n % 2 == 1; - * }); - * // => 3 - */ - var findLast = createFind(findLastIndex); - - /** - * Creates a flattened array of values by running each element in `collection` - * thru `iteratee` and flattening the mapped results. The iteratee is invoked - * with three arguments: (value, index|key, collection). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new flattened array. - * @example - * - * function duplicate(n) { - * return [n, n]; - * } - * - * _.flatMap([1, 2], duplicate); - * // => [1, 1, 2, 2] - */ - function flatMap(collection, iteratee) { - return baseFlatten(map(collection, iteratee), 1); - } - - /** - * This method is like `_.flatMap` except that it recursively flattens the - * mapped results. - * - * @static - * @memberOf _ - * @since 4.7.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new flattened array. - * @example - * - * function duplicate(n) { - * return [[[n, n]]]; - * } - * - * _.flatMapDeep([1, 2], duplicate); - * // => [1, 1, 2, 2] - */ - function flatMapDeep(collection, iteratee) { - return baseFlatten(map(collection, iteratee), INFINITY); - } - - /** - * This method is like `_.flatMap` except that it recursively flattens the - * mapped results up to `depth` times. - * - * @static - * @memberOf _ - * @since 4.7.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @param {number} [depth=1] The maximum recursion depth. - * @returns {Array} Returns the new flattened array. - * @example - * - * function duplicate(n) { - * return [[[n, n]]]; - * } - * - * _.flatMapDepth([1, 2], duplicate, 2); - * // => [[1, 1], [2, 2]] - */ - function flatMapDepth(collection, iteratee, depth) { - depth = depth === undefined ? 1 : toInteger(depth); - return baseFlatten(map(collection, iteratee), depth); - } - - /** - * Iterates over elements of `collection` and invokes `iteratee` for each element. - * The iteratee is invoked with three arguments: (value, index|key, collection). - * Iteratee functions may exit iteration early by explicitly returning `false`. - * - * **Note:** As with other "Collections" methods, objects with a "length" - * property are iterated like arrays. To avoid this behavior use `_.forIn` - * or `_.forOwn` for object iteration. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @alias each - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. - * @see _.forEachRight - * @example - * - * _.forEach([1, 2], function(value) { - * console.log(value); - * }); - * // => Logs `1` then `2`. - * - * _.forEach({ 'a': 1, 'b': 2 }, function(value, key) { - * console.log(key); - * }); - * // => Logs 'a' then 'b' (iteration order is not guaranteed). - */ - function forEach(collection, iteratee) { - var func = isArray(collection) ? arrayEach : baseEach; - return func(collection, getIteratee(iteratee, 3)); - } - - /** - * This method is like `_.forEach` except that it iterates over elements of - * `collection` from right to left. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @alias eachRight - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array|Object} Returns `collection`. - * @see _.forEach - * @example - * - * _.forEachRight([1, 2], function(value) { - * console.log(value); - * }); - * // => Logs `2` then `1`. - */ - function forEachRight(collection, iteratee) { - var func = isArray(collection) ? arrayEachRight : baseEachRight; - return func(collection, getIteratee(iteratee, 3)); - } - - /** - * Creates an object composed of keys generated from the results of running - * each element of `collection` thru `iteratee`. The order of grouped values - * is determined by the order they occur in `collection`. The corresponding - * value of each key is an array of elements responsible for generating the - * key. The iteratee is invoked with one argument: (value). - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The iteratee to transform keys. - * @returns {Object} Returns the composed aggregate object. - * @example - * - * _.groupBy([6.1, 4.2, 6.3], Math.floor); - * // => { '4': [4.2], '6': [6.1, 6.3] } - * - * // The `_.property` iteratee shorthand. - * _.groupBy(['one', 'two', 'three'], 'length'); - * // => { '3': ['one', 'two'], '5': ['three'] } - */ - var groupBy = createAggregator(function(result, value, key) { - if (hasOwnProperty.call(result, key)) { - result[key].push(value); - } else { - baseAssignValue(result, key, [value]); - } - }); - - /** - * Checks if `value` is in `collection`. If `collection` is a string, it's - * checked for a substring of `value`, otherwise - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * is used for equality comparisons. If `fromIndex` is negative, it's used as - * the offset from the end of `collection`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object|string} collection The collection to inspect. - * @param {*} value The value to search for. - * @param {number} [fromIndex=0] The index to search from. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.reduce`. - * @returns {boolean} Returns `true` if `value` is found, else `false`. - * @example - * - * _.includes([1, 2, 3], 1); - * // => true - * - * _.includes([1, 2, 3], 1, 2); - * // => false - * - * _.includes({ 'a': 1, 'b': 2 }, 1); - * // => true - * - * _.includes('abcd', 'bc'); - * // => true - */ - function includes(collection, value, fromIndex, guard) { - collection = isArrayLike(collection) ? collection : values(collection); - fromIndex = (fromIndex && !guard) ? toInteger(fromIndex) : 0; - - var length = collection.length; - if (fromIndex < 0) { - fromIndex = nativeMax(length + fromIndex, 0); - } - return isString(collection) - ? (fromIndex <= length && collection.indexOf(value, fromIndex) > -1) - : (!!length && baseIndexOf(collection, value, fromIndex) > -1); - } - - /** - * Invokes the method at `path` of each element in `collection`, returning - * an array of the results of each invoked method. Any additional arguments - * are provided to each invoked method. If `path` is a function, it's invoked - * for, and `this` bound to, each element in `collection`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Array|Function|string} path The path of the method to invoke or - * the function invoked per iteration. - * @param {...*} [args] The arguments to invoke each method with. - * @returns {Array} Returns the array of results. - * @example - * - * _.invokeMap([[5, 1, 7], [3, 2, 1]], 'sort'); - * // => [[1, 5, 7], [1, 2, 3]] - * - * _.invokeMap([123, 456], String.prototype.split, ''); - * // => [['1', '2', '3'], ['4', '5', '6']] - */ - var invokeMap = baseRest(function(collection, path, args) { - var index = -1, - isFunc = typeof path == 'function', - result = isArrayLike(collection) ? Array(collection.length) : []; - - baseEach(collection, function(value) { - result[++index] = isFunc ? apply(path, value, args) : baseInvoke(value, path, args); - }); - return result; - }); - - /** - * Creates an object composed of keys generated from the results of running - * each element of `collection` thru `iteratee`. The corresponding value of - * each key is the last element responsible for generating the key. The - * iteratee is invoked with one argument: (value). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The iteratee to transform keys. - * @returns {Object} Returns the composed aggregate object. - * @example - * - * var array = [ - * { 'dir': 'left', 'code': 97 }, - * { 'dir': 'right', 'code': 100 } - * ]; - * - * _.keyBy(array, function(o) { - * return String.fromCharCode(o.code); - * }); - * // => { 'a': { 'dir': 'left', 'code': 97 }, 'd': { 'dir': 'right', 'code': 100 } } - * - * _.keyBy(array, 'dir'); - * // => { 'left': { 'dir': 'left', 'code': 97 }, 'right': { 'dir': 'right', 'code': 100 } } - */ - var keyBy = createAggregator(function(result, value, key) { - baseAssignValue(result, key, value); - }); - - /** - * Creates an array of values by running each element in `collection` thru - * `iteratee`. The iteratee is invoked with three arguments: - * (value, index|key, collection). - * - * Many lodash methods are guarded to work as iteratees for methods like - * `_.every`, `_.filter`, `_.map`, `_.mapValues`, `_.reject`, and `_.some`. - * - * The guarded methods are: - * `ary`, `chunk`, `curry`, `curryRight`, `drop`, `dropRight`, `every`, - * `fill`, `invert`, `parseInt`, `random`, `range`, `rangeRight`, `repeat`, - * `sampleSize`, `slice`, `some`, `sortBy`, `split`, `take`, `takeRight`, - * `template`, `trim`, `trimEnd`, `trimStart`, and `words` - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new mapped array. - * @example - * - * function square(n) { - * return n * n; - * } - * - * _.map([4, 8], square); - * // => [16, 64] - * - * _.map({ 'a': 4, 'b': 8 }, square); - * // => [16, 64] (iteration order is not guaranteed) - * - * var users = [ - * { 'user': 'barney' }, - * { 'user': 'fred' } - * ]; - * - * // The `_.property` iteratee shorthand. - * _.map(users, 'user'); - * // => ['barney', 'fred'] - */ - function map(collection, iteratee) { - var func = isArray(collection) ? arrayMap : baseMap; - return func(collection, getIteratee(iteratee, 3)); - } - - /** - * This method is like `_.sortBy` except that it allows specifying the sort - * orders of the iteratees to sort by. If `orders` is unspecified, all values - * are sorted in ascending order. Otherwise, specify an order of "desc" for - * descending or "asc" for ascending sort order of corresponding values. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Array[]|Function[]|Object[]|string[]} [iteratees=[_.identity]] - * The iteratees to sort by. - * @param {string[]} [orders] The sort orders of `iteratees`. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.reduce`. - * @returns {Array} Returns the new sorted array. - * @example - * - * var users = [ - * { 'user': 'fred', 'age': 48 }, - * { 'user': 'barney', 'age': 34 }, - * { 'user': 'fred', 'age': 40 }, - * { 'user': 'barney', 'age': 36 } - * ]; - * - * // Sort by `user` in ascending order and by `age` in descending order. - * _.orderBy(users, ['user', 'age'], ['asc', 'desc']); - * // => objects for [['barney', 36], ['barney', 34], ['fred', 48], ['fred', 40]] - */ - function orderBy(collection, iteratees, orders, guard) { - if (collection == null) { - return []; - } - if (!isArray(iteratees)) { - iteratees = iteratees == null ? [] : [iteratees]; - } - orders = guard ? undefined : orders; - if (!isArray(orders)) { - orders = orders == null ? [] : [orders]; - } - return baseOrderBy(collection, iteratees, orders); - } - - /** - * Creates an array of elements split into two groups, the first of which - * contains elements `predicate` returns truthy for, the second of which - * contains elements `predicate` returns falsey for. The predicate is - * invoked with one argument: (value). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the array of grouped elements. - * @example - * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': false }, - * { 'user': 'fred', 'age': 40, 'active': true }, - * { 'user': 'pebbles', 'age': 1, 'active': false } - * ]; - * - * _.partition(users, function(o) { return o.active; }); - * // => objects for [['fred'], ['barney', 'pebbles']] - * - * // The `_.matches` iteratee shorthand. - * _.partition(users, { 'age': 1, 'active': false }); - * // => objects for [['pebbles'], ['barney', 'fred']] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.partition(users, ['active', false]); - * // => objects for [['barney', 'pebbles'], ['fred']] - * - * // The `_.property` iteratee shorthand. - * _.partition(users, 'active'); - * // => objects for [['fred'], ['barney', 'pebbles']] - */ - var partition = createAggregator(function(result, value, key) { - result[key ? 0 : 1].push(value); - }, function() { return [[], []]; }); - - /** - * Reduces `collection` to a value which is the accumulated result of running - * each element in `collection` thru `iteratee`, where each successive - * invocation is supplied the return value of the previous. If `accumulator` - * is not given, the first element of `collection` is used as the initial - * value. The iteratee is invoked with four arguments: - * (accumulator, value, index|key, collection). - * - * Many lodash methods are guarded to work as iteratees for methods like - * `_.reduce`, `_.reduceRight`, and `_.transform`. - * - * The guarded methods are: - * `assign`, `defaults`, `defaultsDeep`, `includes`, `merge`, `orderBy`, - * and `sortBy` - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @returns {*} Returns the accumulated value. - * @see _.reduceRight - * @example - * - * _.reduce([1, 2], function(sum, n) { - * return sum + n; - * }, 0); - * // => 3 - * - * _.reduce({ 'a': 1, 'b': 2, 'c': 1 }, function(result, value, key) { - * (result[value] || (result[value] = [])).push(key); - * return result; - * }, {}); - * // => { '1': ['a', 'c'], '2': ['b'] } (iteration order is not guaranteed) - */ - function reduce(collection, iteratee, accumulator) { - var func = isArray(collection) ? arrayReduce : baseReduce, - initAccum = arguments.length < 3; - - return func(collection, getIteratee(iteratee, 4), accumulator, initAccum, baseEach); - } - - /** - * This method is like `_.reduce` except that it iterates over elements of - * `collection` from right to left. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @param {*} [accumulator] The initial value. - * @returns {*} Returns the accumulated value. - * @see _.reduce - * @example - * - * var array = [[0, 1], [2, 3], [4, 5]]; - * - * _.reduceRight(array, function(flattened, other) { - * return flattened.concat(other); - * }, []); - * // => [4, 5, 2, 3, 0, 1] - */ - function reduceRight(collection, iteratee, accumulator) { - var func = isArray(collection) ? arrayReduceRight : baseReduce, - initAccum = arguments.length < 3; - - return func(collection, getIteratee(iteratee, 4), accumulator, initAccum, baseEachRight); - } - - /** - * The opposite of `_.filter`; this method returns the elements of `collection` - * that `predicate` does **not** return truthy for. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {Array} Returns the new filtered array. - * @see _.filter - * @example - * - * var users = [ - * { 'user': 'barney', 'age': 36, 'active': false }, - * { 'user': 'fred', 'age': 40, 'active': true } - * ]; - * - * _.reject(users, function(o) { return !o.active; }); - * // => objects for ['fred'] - * - * // The `_.matches` iteratee shorthand. - * _.reject(users, { 'age': 40, 'active': true }); - * // => objects for ['barney'] - * - * // The `_.matchesProperty` iteratee shorthand. - * _.reject(users, ['active', false]); - * // => objects for ['fred'] - * - * // The `_.property` iteratee shorthand. - * _.reject(users, 'active'); - * // => objects for ['barney'] - */ - function reject(collection, predicate) { - var func = isArray(collection) ? arrayFilter : baseFilter; - return func(collection, negate(getIteratee(predicate, 3))); - } - - /** - * Gets a random element from `collection`. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Collection - * @param {Array|Object} collection The collection to sample. - * @returns {*} Returns the random element. - * @example - * - * _.sample([1, 2, 3, 4]); - * // => 2 - */ - function sample(collection) { - var func = isArray(collection) ? arraySample : baseSample; - return func(collection); - } - - /** - * Gets `n` random elements at unique keys from `collection` up to the - * size of `collection`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Collection - * @param {Array|Object} collection The collection to sample. - * @param {number} [n=1] The number of elements to sample. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Array} Returns the random elements. - * @example - * - * _.sampleSize([1, 2, 3], 2); - * // => [3, 1] - * - * _.sampleSize([1, 2, 3], 4); - * // => [2, 3, 1] - */ - function sampleSize(collection, n, guard) { - if ((guard ? isIterateeCall(collection, n, guard) : n === undefined)) { - n = 1; - } else { - n = toInteger(n); - } - var func = isArray(collection) ? arraySampleSize : baseSampleSize; - return func(collection, n); - } - - /** - * Creates an array of shuffled values, using a version of the - * [Fisher-Yates shuffle](https://en.wikipedia.org/wiki/Fisher-Yates_shuffle). - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to shuffle. - * @returns {Array} Returns the new shuffled array. - * @example - * - * _.shuffle([1, 2, 3, 4]); - * // => [4, 1, 3, 2] - */ - function shuffle(collection) { - var func = isArray(collection) ? arrayShuffle : baseShuffle; - return func(collection); - } - - /** - * Gets the size of `collection` by returning its length for array-like - * values or the number of own enumerable string keyed properties for objects. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object|string} collection The collection to inspect. - * @returns {number} Returns the collection size. - * @example - * - * _.size([1, 2, 3]); - * // => 3 - * - * _.size({ 'a': 1, 'b': 2 }); - * // => 2 - * - * _.size('pebbles'); - * // => 7 - */ - function size(collection) { - if (collection == null) { - return 0; - } - if (isArrayLike(collection)) { - return isString(collection) ? stringSize(collection) : collection.length; - } - var tag = getTag(collection); - if (tag == mapTag || tag == setTag) { - return collection.size; - } - return baseKeys(collection).length; - } - - /** - * Checks if `predicate` returns truthy for **any** element of `collection`. - * Iteration is stopped once `predicate` returns truthy. The predicate is - * invoked with three arguments: (value, index|key, collection). - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {boolean} Returns `true` if any element passes the predicate check, - * else `false`. - * @example - * - * _.some([null, 0, 'yes', false], Boolean); - * // => true - * - * var users = [ - * { 'user': 'barney', 'active': true }, - * { 'user': 'fred', 'active': false } - * ]; - * - * // The `_.matches` iteratee shorthand. - * _.some(users, { 'user': 'barney', 'active': false }); - * // => false - * - * // The `_.matchesProperty` iteratee shorthand. - * _.some(users, ['active', false]); - * // => true - * - * // The `_.property` iteratee shorthand. - * _.some(users, 'active'); - * // => true - */ - function some(collection, predicate, guard) { - var func = isArray(collection) ? arraySome : baseSome; - if (guard && isIterateeCall(collection, predicate, guard)) { - predicate = undefined; - } - return func(collection, getIteratee(predicate, 3)); - } - - /** - * Creates an array of elements, sorted in ascending order by the results of - * running each element in a collection thru each iteratee. This method - * performs a stable sort, that is, it preserves the original sort order of - * equal elements. The iteratees are invoked with one argument: (value). - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Collection - * @param {Array|Object} collection The collection to iterate over. - * @param {...(Function|Function[])} [iteratees=[_.identity]] - * The iteratees to sort by. - * @returns {Array} Returns the new sorted array. - * @example - * - * var users = [ - * { 'user': 'fred', 'age': 48 }, - * { 'user': 'barney', 'age': 36 }, - * { 'user': 'fred', 'age': 40 }, - * { 'user': 'barney', 'age': 34 } - * ]; - * - * _.sortBy(users, [function(o) { return o.user; }]); - * // => objects for [['barney', 36], ['barney', 34], ['fred', 48], ['fred', 40]] - * - * _.sortBy(users, ['user', 'age']); - * // => objects for [['barney', 34], ['barney', 36], ['fred', 40], ['fred', 48]] - */ - var sortBy = baseRest(function(collection, iteratees) { - if (collection == null) { - return []; - } - var length = iteratees.length; - if (length > 1 && isIterateeCall(collection, iteratees[0], iteratees[1])) { - iteratees = []; - } else if (length > 2 && isIterateeCall(iteratees[0], iteratees[1], iteratees[2])) { - iteratees = [iteratees[0]]; - } - return baseOrderBy(collection, baseFlatten(iteratees, 1), []); - }); - - /*------------------------------------------------------------------------*/ - - /** - * Gets the timestamp of the number of milliseconds that have elapsed since - * the Unix epoch (1 January 1970 00:00:00 UTC). - * - * @static - * @memberOf _ - * @since 2.4.0 - * @category Date - * @returns {number} Returns the timestamp. - * @example - * - * _.defer(function(stamp) { - * console.log(_.now() - stamp); - * }, _.now()); - * // => Logs the number of milliseconds it took for the deferred invocation. - */ - var now = ctxNow || function() { - return root.Date.now(); - }; - - /*------------------------------------------------------------------------*/ - - /** - * The opposite of `_.before`; this method creates a function that invokes - * `func` once it's called `n` or more times. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {number} n The number of calls before `func` is invoked. - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new restricted function. - * @example - * - * var saves = ['profile', 'settings']; - * - * var done = _.after(saves.length, function() { - * console.log('done saving!'); - * }); - * - * _.forEach(saves, function(type) { - * asyncSave({ 'type': type, 'complete': done }); - * }); - * // => Logs 'done saving!' after the two async saves have completed. - */ - function after(n, func) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - n = toInteger(n); - return function() { - if (--n < 1) { - return func.apply(this, arguments); - } - }; - } - - /** - * Creates a function that invokes `func`, with up to `n` arguments, - * ignoring any additional arguments. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {Function} func The function to cap arguments for. - * @param {number} [n=func.length] The arity cap. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the new capped function. - * @example - * - * _.map(['6', '8', '10'], _.ary(parseInt, 1)); - * // => [6, 8, 10] - */ - function ary(func, n, guard) { - n = guard ? undefined : n; - n = (func && n == null) ? func.length : n; - return createWrap(func, WRAP_ARY_FLAG, undefined, undefined, undefined, undefined, n); - } - - /** - * Creates a function that invokes `func`, with the `this` binding and arguments - * of the created function, while it's called less than `n` times. Subsequent - * calls to the created function return the result of the last `func` invocation. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {number} n The number of calls at which `func` is no longer invoked. - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new restricted function. - * @example - * - * jQuery(element).on('click', _.before(5, addContactToList)); - * // => Allows adding up to 4 contacts to the list. - */ - function before(n, func) { - var result; - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - n = toInteger(n); - return function() { - if (--n > 0) { - result = func.apply(this, arguments); - } - if (n <= 1) { - func = undefined; - } - return result; - }; - } - - /** - * Creates a function that invokes `func` with the `this` binding of `thisArg` - * and `partials` prepended to the arguments it receives. - * - * The `_.bind.placeholder` value, which defaults to `_` in monolithic builds, - * may be used as a placeholder for partially applied arguments. - * - * **Note:** Unlike native `Function#bind`, this method doesn't set the "length" - * property of bound functions. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to bind. - * @param {*} thisArg The `this` binding of `func`. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new bound function. - * @example - * - * function greet(greeting, punctuation) { - * return greeting + ' ' + this.user + punctuation; - * } - * - * var object = { 'user': 'fred' }; - * - * var bound = _.bind(greet, object, 'hi'); - * bound('!'); - * // => 'hi fred!' - * - * // Bound with placeholders. - * var bound = _.bind(greet, object, _, '!'); - * bound('hi'); - * // => 'hi fred!' - */ - var bind = baseRest(function(func, thisArg, partials) { - var bitmask = WRAP_BIND_FLAG; - if (partials.length) { - var holders = replaceHolders(partials, getHolder(bind)); - bitmask |= WRAP_PARTIAL_FLAG; - } - return createWrap(func, bitmask, thisArg, partials, holders); - }); - - /** - * Creates a function that invokes the method at `object[key]` with `partials` - * prepended to the arguments it receives. - * - * This method differs from `_.bind` by allowing bound functions to reference - * methods that may be redefined or don't yet exist. See - * [Peter Michaux's article](http://peter.michaux.ca/articles/lazy-function-definition-pattern) - * for more details. - * - * The `_.bindKey.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for partially applied arguments. - * - * @static - * @memberOf _ - * @since 0.10.0 - * @category Function - * @param {Object} object The object to invoke the method on. - * @param {string} key The key of the method. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new bound function. - * @example - * - * var object = { - * 'user': 'fred', - * 'greet': function(greeting, punctuation) { - * return greeting + ' ' + this.user + punctuation; - * } - * }; - * - * var bound = _.bindKey(object, 'greet', 'hi'); - * bound('!'); - * // => 'hi fred!' - * - * object.greet = function(greeting, punctuation) { - * return greeting + 'ya ' + this.user + punctuation; - * }; - * - * bound('!'); - * // => 'hiya fred!' - * - * // Bound with placeholders. - * var bound = _.bindKey(object, 'greet', _, '!'); - * bound('hi'); - * // => 'hiya fred!' - */ - var bindKey = baseRest(function(object, key, partials) { - var bitmask = WRAP_BIND_FLAG | WRAP_BIND_KEY_FLAG; - if (partials.length) { - var holders = replaceHolders(partials, getHolder(bindKey)); - bitmask |= WRAP_PARTIAL_FLAG; - } - return createWrap(key, bitmask, object, partials, holders); - }); - - /** - * Creates a function that accepts arguments of `func` and either invokes - * `func` returning its result, if at least `arity` number of arguments have - * been provided, or returns a function that accepts the remaining `func` - * arguments, and so on. The arity of `func` may be specified if `func.length` - * is not sufficient. - * - * The `_.curry.placeholder` value, which defaults to `_` in monolithic builds, - * may be used as a placeholder for provided arguments. - * - * **Note:** This method doesn't set the "length" property of curried functions. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Function - * @param {Function} func The function to curry. - * @param {number} [arity=func.length] The arity of `func`. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the new curried function. - * @example - * - * var abc = function(a, b, c) { - * return [a, b, c]; - * }; - * - * var curried = _.curry(abc); - * - * curried(1)(2)(3); - * // => [1, 2, 3] - * - * curried(1, 2)(3); - * // => [1, 2, 3] - * - * curried(1, 2, 3); - * // => [1, 2, 3] - * - * // Curried with placeholders. - * curried(1)(_, 3)(2); - * // => [1, 2, 3] - */ - function curry(func, arity, guard) { - arity = guard ? undefined : arity; - var result = createWrap(func, WRAP_CURRY_FLAG, undefined, undefined, undefined, undefined, undefined, arity); - result.placeholder = curry.placeholder; - return result; - } - - /** - * This method is like `_.curry` except that arguments are applied to `func` - * in the manner of `_.partialRight` instead of `_.partial`. - * - * The `_.curryRight.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for provided arguments. - * - * **Note:** This method doesn't set the "length" property of curried functions. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {Function} func The function to curry. - * @param {number} [arity=func.length] The arity of `func`. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the new curried function. - * @example - * - * var abc = function(a, b, c) { - * return [a, b, c]; - * }; - * - * var curried = _.curryRight(abc); - * - * curried(3)(2)(1); - * // => [1, 2, 3] - * - * curried(2, 3)(1); - * // => [1, 2, 3] - * - * curried(1, 2, 3); - * // => [1, 2, 3] - * - * // Curried with placeholders. - * curried(3)(1, _)(2); - * // => [1, 2, 3] - */ - function curryRight(func, arity, guard) { - arity = guard ? undefined : arity; - var result = createWrap(func, WRAP_CURRY_RIGHT_FLAG, undefined, undefined, undefined, undefined, undefined, arity); - result.placeholder = curryRight.placeholder; - return result; - } - - /** - * Creates a debounced function that delays invoking `func` until after `wait` - * milliseconds have elapsed since the last time the debounced function was - * invoked. The debounced function comes with a `cancel` method to cancel - * delayed `func` invocations and a `flush` method to immediately invoke them. - * Provide `options` to indicate whether `func` should be invoked on the - * leading and/or trailing edge of the `wait` timeout. The `func` is invoked - * with the last arguments provided to the debounced function. Subsequent - * calls to the debounced function return the result of the last `func` - * invocation. - * - * **Note:** If `leading` and `trailing` options are `true`, `func` is - * invoked on the trailing edge of the timeout only if the debounced function - * is invoked more than once during the `wait` timeout. - * - * If `wait` is `0` and `leading` is `false`, `func` invocation is deferred - * until to the next tick, similar to `setTimeout` with a timeout of `0`. - * - * See [David Corbacho's article](https://css-tricks.com/debouncing-throttling-explained-examples/) - * for details over the differences between `_.debounce` and `_.throttle`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to debounce. - * @param {number} [wait=0] The number of milliseconds to delay. - * @param {Object} [options={}] The options object. - * @param {boolean} [options.leading=false] - * Specify invoking on the leading edge of the timeout. - * @param {number} [options.maxWait] - * The maximum time `func` is allowed to be delayed before it's invoked. - * @param {boolean} [options.trailing=true] - * Specify invoking on the trailing edge of the timeout. - * @returns {Function} Returns the new debounced function. - * @example - * - * // Avoid costly calculations while the window size is in flux. - * jQuery(window).on('resize', _.debounce(calculateLayout, 150)); - * - * // Invoke `sendMail` when clicked, debouncing subsequent calls. - * jQuery(element).on('click', _.debounce(sendMail, 300, { - * 'leading': true, - * 'trailing': false - * })); - * - * // Ensure `batchLog` is invoked once after 1 second of debounced calls. - * var debounced = _.debounce(batchLog, 250, { 'maxWait': 1000 }); - * var source = new EventSource('/stream'); - * jQuery(source).on('message', debounced); - * - * // Cancel the trailing debounced invocation. - * jQuery(window).on('popstate', debounced.cancel); - */ - function debounce(func, wait, options) { - var lastArgs, - lastThis, - maxWait, - result, - timerId, - lastCallTime, - lastInvokeTime = 0, - leading = false, - maxing = false, - trailing = true; - - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - wait = toNumber(wait) || 0; - if (isObject(options)) { - leading = !!options.leading; - maxing = 'maxWait' in options; - maxWait = maxing ? nativeMax(toNumber(options.maxWait) || 0, wait) : maxWait; - trailing = 'trailing' in options ? !!options.trailing : trailing; - } - - function invokeFunc(time) { - var args = lastArgs, - thisArg = lastThis; - - lastArgs = lastThis = undefined; - lastInvokeTime = time; - result = func.apply(thisArg, args); - return result; - } - - function leadingEdge(time) { - // Reset any `maxWait` timer. - lastInvokeTime = time; - // Start the timer for the trailing edge. - timerId = setTimeout(timerExpired, wait); - // Invoke the leading edge. - return leading ? invokeFunc(time) : result; - } - - function remainingWait(time) { - var timeSinceLastCall = time - lastCallTime, - timeSinceLastInvoke = time - lastInvokeTime, - timeWaiting = wait - timeSinceLastCall; - - return maxing - ? nativeMin(timeWaiting, maxWait - timeSinceLastInvoke) - : timeWaiting; - } - - function shouldInvoke(time) { - var timeSinceLastCall = time - lastCallTime, - timeSinceLastInvoke = time - lastInvokeTime; - - // Either this is the first call, activity has stopped and we're at the - // trailing edge, the system time has gone backwards and we're treating - // it as the trailing edge, or we've hit the `maxWait` limit. - return (lastCallTime === undefined || (timeSinceLastCall >= wait) || - (timeSinceLastCall < 0) || (maxing && timeSinceLastInvoke >= maxWait)); - } - - function timerExpired() { - var time = now(); - if (shouldInvoke(time)) { - return trailingEdge(time); - } - // Restart the timer. - timerId = setTimeout(timerExpired, remainingWait(time)); - } - - function trailingEdge(time) { - timerId = undefined; - - // Only invoke if we have `lastArgs` which means `func` has been - // debounced at least once. - if (trailing && lastArgs) { - return invokeFunc(time); - } - lastArgs = lastThis = undefined; - return result; - } - - function cancel() { - if (timerId !== undefined) { - clearTimeout(timerId); - } - lastInvokeTime = 0; - lastArgs = lastCallTime = lastThis = timerId = undefined; - } - - function flush() { - return timerId === undefined ? result : trailingEdge(now()); - } - - function debounced() { - var time = now(), - isInvoking = shouldInvoke(time); - - lastArgs = arguments; - lastThis = this; - lastCallTime = time; - - if (isInvoking) { - if (timerId === undefined) { - return leadingEdge(lastCallTime); - } - if (maxing) { - // Handle invocations in a tight loop. - clearTimeout(timerId); - timerId = setTimeout(timerExpired, wait); - return invokeFunc(lastCallTime); - } - } - if (timerId === undefined) { - timerId = setTimeout(timerExpired, wait); - } - return result; - } - debounced.cancel = cancel; - debounced.flush = flush; - return debounced; - } - - /** - * Defers invoking the `func` until the current call stack has cleared. Any - * additional arguments are provided to `func` when it's invoked. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to defer. - * @param {...*} [args] The arguments to invoke `func` with. - * @returns {number} Returns the timer id. - * @example - * - * _.defer(function(text) { - * console.log(text); - * }, 'deferred'); - * // => Logs 'deferred' after one millisecond. - */ - var defer = baseRest(function(func, args) { - return baseDelay(func, 1, args); - }); - - /** - * Invokes `func` after `wait` milliseconds. Any additional arguments are - * provided to `func` when it's invoked. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to delay. - * @param {number} wait The number of milliseconds to delay invocation. - * @param {...*} [args] The arguments to invoke `func` with. - * @returns {number} Returns the timer id. - * @example - * - * _.delay(function(text) { - * console.log(text); - * }, 1000, 'later'); - * // => Logs 'later' after one second. - */ - var delay = baseRest(function(func, wait, args) { - return baseDelay(func, toNumber(wait) || 0, args); - }); - - /** - * Creates a function that invokes `func` with arguments reversed. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Function - * @param {Function} func The function to flip arguments for. - * @returns {Function} Returns the new flipped function. - * @example - * - * var flipped = _.flip(function() { - * return _.toArray(arguments); - * }); - * - * flipped('a', 'b', 'c', 'd'); - * // => ['d', 'c', 'b', 'a'] - */ - function flip(func) { - return createWrap(func, WRAP_FLIP_FLAG); - } - - /** - * Creates a function that memoizes the result of `func`. If `resolver` is - * provided, it determines the cache key for storing the result based on the - * arguments provided to the memoized function. By default, the first argument - * provided to the memoized function is used as the map cache key. The `func` - * is invoked with the `this` binding of the memoized function. - * - * **Note:** The cache is exposed as the `cache` property on the memoized - * function. Its creation may be customized by replacing the `_.memoize.Cache` - * constructor with one whose instances implement the - * [`Map`](http://ecma-international.org/ecma-262/7.0/#sec-properties-of-the-map-prototype-object) - * method interface of `clear`, `delete`, `get`, `has`, and `set`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to have its output memoized. - * @param {Function} [resolver] The function to resolve the cache key. - * @returns {Function} Returns the new memoized function. - * @example - * - * var object = { 'a': 1, 'b': 2 }; - * var other = { 'c': 3, 'd': 4 }; - * - * var values = _.memoize(_.values); - * values(object); - * // => [1, 2] - * - * values(other); - * // => [3, 4] - * - * object.a = 2; - * values(object); - * // => [1, 2] - * - * // Modify the result cache. - * values.cache.set(object, ['a', 'b']); - * values(object); - * // => ['a', 'b'] - * - * // Replace `_.memoize.Cache`. - * _.memoize.Cache = WeakMap; - */ - function memoize(func, resolver) { - if (typeof func != 'function' || (resolver != null && typeof resolver != 'function')) { - throw new TypeError(FUNC_ERROR_TEXT); - } - var memoized = function() { - var args = arguments, - key = resolver ? resolver.apply(this, args) : args[0], - cache = memoized.cache; - - if (cache.has(key)) { - return cache.get(key); - } - var result = func.apply(this, args); - memoized.cache = cache.set(key, result) || cache; - return result; - }; - memoized.cache = new (memoize.Cache || MapCache); - return memoized; - } - - // Expose `MapCache`. - memoize.Cache = MapCache; - - /** - * Creates a function that negates the result of the predicate `func`. The - * `func` predicate is invoked with the `this` binding and arguments of the - * created function. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {Function} predicate The predicate to negate. - * @returns {Function} Returns the new negated function. - * @example - * - * function isEven(n) { - * return n % 2 == 0; - * } - * - * _.filter([1, 2, 3, 4, 5, 6], _.negate(isEven)); - * // => [1, 3, 5] - */ - function negate(predicate) { - if (typeof predicate != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - return function() { - var args = arguments; - switch (args.length) { - case 0: return !predicate.call(this); - case 1: return !predicate.call(this, args[0]); - case 2: return !predicate.call(this, args[0], args[1]); - case 3: return !predicate.call(this, args[0], args[1], args[2]); - } - return !predicate.apply(this, args); - }; - } - - /** - * Creates a function that is restricted to invoking `func` once. Repeat calls - * to the function return the value of the first invocation. The `func` is - * invoked with the `this` binding and arguments of the created function. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to restrict. - * @returns {Function} Returns the new restricted function. - * @example - * - * var initialize = _.once(createApplication); - * initialize(); - * initialize(); - * // => `createApplication` is invoked once - */ - function once(func) { - return before(2, func); - } - - /** - * Creates a function that invokes `func` with its arguments transformed. - * - * @static - * @since 4.0.0 - * @memberOf _ - * @category Function - * @param {Function} func The function to wrap. - * @param {...(Function|Function[])} [transforms=[_.identity]] - * The argument transforms. - * @returns {Function} Returns the new function. - * @example - * - * function doubled(n) { - * return n * 2; - * } - * - * function square(n) { - * return n * n; - * } - * - * var func = _.overArgs(function(x, y) { - * return [x, y]; - * }, [square, doubled]); - * - * func(9, 3); - * // => [81, 6] - * - * func(10, 5); - * // => [100, 10] - */ - var overArgs = castRest(function(func, transforms) { - transforms = (transforms.length == 1 && isArray(transforms[0])) - ? arrayMap(transforms[0], baseUnary(getIteratee())) - : arrayMap(baseFlatten(transforms, 1), baseUnary(getIteratee())); - - var funcsLength = transforms.length; - return baseRest(function(args) { - var index = -1, - length = nativeMin(args.length, funcsLength); - - while (++index < length) { - args[index] = transforms[index].call(this, args[index]); - } - return apply(func, this, args); - }); - }); - - /** - * Creates a function that invokes `func` with `partials` prepended to the - * arguments it receives. This method is like `_.bind` except it does **not** - * alter the `this` binding. - * - * The `_.partial.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for partially applied arguments. - * - * **Note:** This method doesn't set the "length" property of partially - * applied functions. - * - * @static - * @memberOf _ - * @since 0.2.0 - * @category Function - * @param {Function} func The function to partially apply arguments to. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new partially applied function. - * @example - * - * function greet(greeting, name) { - * return greeting + ' ' + name; - * } - * - * var sayHelloTo = _.partial(greet, 'hello'); - * sayHelloTo('fred'); - * // => 'hello fred' - * - * // Partially applied with placeholders. - * var greetFred = _.partial(greet, _, 'fred'); - * greetFred('hi'); - * // => 'hi fred' - */ - var partial = baseRest(function(func, partials) { - var holders = replaceHolders(partials, getHolder(partial)); - return createWrap(func, WRAP_PARTIAL_FLAG, undefined, partials, holders); - }); - - /** - * This method is like `_.partial` except that partially applied arguments - * are appended to the arguments it receives. - * - * The `_.partialRight.placeholder` value, which defaults to `_` in monolithic - * builds, may be used as a placeholder for partially applied arguments. - * - * **Note:** This method doesn't set the "length" property of partially - * applied functions. - * - * @static - * @memberOf _ - * @since 1.0.0 - * @category Function - * @param {Function} func The function to partially apply arguments to. - * @param {...*} [partials] The arguments to be partially applied. - * @returns {Function} Returns the new partially applied function. - * @example - * - * function greet(greeting, name) { - * return greeting + ' ' + name; - * } - * - * var greetFred = _.partialRight(greet, 'fred'); - * greetFred('hi'); - * // => 'hi fred' - * - * // Partially applied with placeholders. - * var sayHelloTo = _.partialRight(greet, 'hello', _); - * sayHelloTo('fred'); - * // => 'hello fred' - */ - var partialRight = baseRest(function(func, partials) { - var holders = replaceHolders(partials, getHolder(partialRight)); - return createWrap(func, WRAP_PARTIAL_RIGHT_FLAG, undefined, partials, holders); - }); - - /** - * Creates a function that invokes `func` with arguments arranged according - * to the specified `indexes` where the argument value at the first index is - * provided as the first argument, the argument value at the second index is - * provided as the second argument, and so on. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Function - * @param {Function} func The function to rearrange arguments for. - * @param {...(number|number[])} indexes The arranged argument indexes. - * @returns {Function} Returns the new function. - * @example - * - * var rearged = _.rearg(function(a, b, c) { - * return [a, b, c]; - * }, [2, 0, 1]); - * - * rearged('b', 'c', 'a') - * // => ['a', 'b', 'c'] - */ - var rearg = flatRest(function(func, indexes) { - return createWrap(func, WRAP_REARG_FLAG, undefined, undefined, undefined, indexes); - }); - - /** - * Creates a function that invokes `func` with the `this` binding of the - * created function and arguments from `start` and beyond provided as - * an array. - * - * **Note:** This method is based on the - * [rest parameter](https://mdn.io/rest_parameters). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Function - * @param {Function} func The function to apply a rest parameter to. - * @param {number} [start=func.length-1] The start position of the rest parameter. - * @returns {Function} Returns the new function. - * @example - * - * var say = _.rest(function(what, names) { - * return what + ' ' + _.initial(names).join(', ') + - * (_.size(names) > 1 ? ', & ' : '') + _.last(names); - * }); - * - * say('hello', 'fred', 'barney', 'pebbles'); - * // => 'hello fred, barney, & pebbles' - */ - function rest(func, start) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - start = start === undefined ? start : toInteger(start); - return baseRest(func, start); - } - - /** - * Creates a function that invokes `func` with the `this` binding of the - * create function and an array of arguments much like - * [`Function#apply`](http://www.ecma-international.org/ecma-262/7.0/#sec-function.prototype.apply). - * - * **Note:** This method is based on the - * [spread operator](https://mdn.io/spread_operator). - * - * @static - * @memberOf _ - * @since 3.2.0 - * @category Function - * @param {Function} func The function to spread arguments over. - * @param {number} [start=0] The start position of the spread. - * @returns {Function} Returns the new function. - * @example - * - * var say = _.spread(function(who, what) { - * return who + ' says ' + what; - * }); - * - * say(['fred', 'hello']); - * // => 'fred says hello' - * - * var numbers = Promise.all([ - * Promise.resolve(40), - * Promise.resolve(36) - * ]); - * - * numbers.then(_.spread(function(x, y) { - * return x + y; - * })); - * // => a Promise of 76 - */ - function spread(func, start) { - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - start = start == null ? 0 : nativeMax(toInteger(start), 0); - return baseRest(function(args) { - var array = args[start], - otherArgs = castSlice(args, 0, start); - - if (array) { - arrayPush(otherArgs, array); - } - return apply(func, this, otherArgs); - }); - } - - /** - * Creates a throttled function that only invokes `func` at most once per - * every `wait` milliseconds. The throttled function comes with a `cancel` - * method to cancel delayed `func` invocations and a `flush` method to - * immediately invoke them. Provide `options` to indicate whether `func` - * should be invoked on the leading and/or trailing edge of the `wait` - * timeout. The `func` is invoked with the last arguments provided to the - * throttled function. Subsequent calls to the throttled function return the - * result of the last `func` invocation. - * - * **Note:** If `leading` and `trailing` options are `true`, `func` is - * invoked on the trailing edge of the timeout only if the throttled function - * is invoked more than once during the `wait` timeout. - * - * If `wait` is `0` and `leading` is `false`, `func` invocation is deferred - * until to the next tick, similar to `setTimeout` with a timeout of `0`. - * - * See [David Corbacho's article](https://css-tricks.com/debouncing-throttling-explained-examples/) - * for details over the differences between `_.throttle` and `_.debounce`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {Function} func The function to throttle. - * @param {number} [wait=0] The number of milliseconds to throttle invocations to. - * @param {Object} [options={}] The options object. - * @param {boolean} [options.leading=true] - * Specify invoking on the leading edge of the timeout. - * @param {boolean} [options.trailing=true] - * Specify invoking on the trailing edge of the timeout. - * @returns {Function} Returns the new throttled function. - * @example - * - * // Avoid excessively updating the position while scrolling. - * jQuery(window).on('scroll', _.throttle(updatePosition, 100)); - * - * // Invoke `renewToken` when the click event is fired, but not more than once every 5 minutes. - * var throttled = _.throttle(renewToken, 300000, { 'trailing': false }); - * jQuery(element).on('click', throttled); - * - * // Cancel the trailing throttled invocation. - * jQuery(window).on('popstate', throttled.cancel); - */ - function throttle(func, wait, options) { - var leading = true, - trailing = true; - - if (typeof func != 'function') { - throw new TypeError(FUNC_ERROR_TEXT); - } - if (isObject(options)) { - leading = 'leading' in options ? !!options.leading : leading; - trailing = 'trailing' in options ? !!options.trailing : trailing; - } - return debounce(func, wait, { - 'leading': leading, - 'maxWait': wait, - 'trailing': trailing - }); - } - - /** - * Creates a function that accepts up to one argument, ignoring any - * additional arguments. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Function - * @param {Function} func The function to cap arguments for. - * @returns {Function} Returns the new capped function. - * @example - * - * _.map(['6', '8', '10'], _.unary(parseInt)); - * // => [6, 8, 10] - */ - function unary(func) { - return ary(func, 1); - } - - /** - * Creates a function that provides `value` to `wrapper` as its first - * argument. Any additional arguments provided to the function are appended - * to those provided to the `wrapper`. The wrapper is invoked with the `this` - * binding of the created function. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Function - * @param {*} value The value to wrap. - * @param {Function} [wrapper=identity] The wrapper function. - * @returns {Function} Returns the new function. - * @example - * - * var p = _.wrap(_.escape, function(func, text) { - * return '

' + func(text) + '

'; - * }); - * - * p('fred, barney, & pebbles'); - * // => '

fred, barney, & pebbles

' - */ - function wrap(value, wrapper) { - return partial(castFunction(wrapper), value); - } - - /*------------------------------------------------------------------------*/ - - /** - * Casts `value` as an array if it's not one. - * - * @static - * @memberOf _ - * @since 4.4.0 - * @category Lang - * @param {*} value The value to inspect. - * @returns {Array} Returns the cast array. - * @example - * - * _.castArray(1); - * // => [1] - * - * _.castArray({ 'a': 1 }); - * // => [{ 'a': 1 }] - * - * _.castArray('abc'); - * // => ['abc'] - * - * _.castArray(null); - * // => [null] - * - * _.castArray(undefined); - * // => [undefined] - * - * _.castArray(); - * // => [] - * - * var array = [1, 2, 3]; - * console.log(_.castArray(array) === array); - * // => true - */ - function castArray() { - if (!arguments.length) { - return []; - } - var value = arguments[0]; - return isArray(value) ? value : [value]; - } - - /** - * Creates a shallow clone of `value`. - * - * **Note:** This method is loosely based on the - * [structured clone algorithm](https://mdn.io/Structured_clone_algorithm) - * and supports cloning arrays, array buffers, booleans, date objects, maps, - * numbers, `Object` objects, regexes, sets, strings, symbols, and typed - * arrays. The own enumerable properties of `arguments` objects are cloned - * as plain objects. An empty object is returned for uncloneable values such - * as error objects, functions, DOM nodes, and WeakMaps. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to clone. - * @returns {*} Returns the cloned value. - * @see _.cloneDeep - * @example - * - * var objects = [{ 'a': 1 }, { 'b': 2 }]; - * - * var shallow = _.clone(objects); - * console.log(shallow[0] === objects[0]); - * // => true - */ - function clone(value) { - return baseClone(value, CLONE_SYMBOLS_FLAG); - } - - /** - * This method is like `_.clone` except that it accepts `customizer` which - * is invoked to produce the cloned value. If `customizer` returns `undefined`, - * cloning is handled by the method instead. The `customizer` is invoked with - * up to four arguments; (value [, index|key, object, stack]). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to clone. - * @param {Function} [customizer] The function to customize cloning. - * @returns {*} Returns the cloned value. - * @see _.cloneDeepWith - * @example - * - * function customizer(value) { - * if (_.isElement(value)) { - * return value.cloneNode(false); - * } - * } - * - * var el = _.cloneWith(document.body, customizer); - * - * console.log(el === document.body); - * // => false - * console.log(el.nodeName); - * // => 'BODY' - * console.log(el.childNodes.length); - * // => 0 - */ - function cloneWith(value, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return baseClone(value, CLONE_SYMBOLS_FLAG, customizer); - } - - /** - * This method is like `_.clone` except that it recursively clones `value`. - * - * @static - * @memberOf _ - * @since 1.0.0 - * @category Lang - * @param {*} value The value to recursively clone. - * @returns {*} Returns the deep cloned value. - * @see _.clone - * @example - * - * var objects = [{ 'a': 1 }, { 'b': 2 }]; - * - * var deep = _.cloneDeep(objects); - * console.log(deep[0] === objects[0]); - * // => false - */ - function cloneDeep(value) { - return baseClone(value, CLONE_DEEP_FLAG | CLONE_SYMBOLS_FLAG); - } - - /** - * This method is like `_.cloneWith` except that it recursively clones `value`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to recursively clone. - * @param {Function} [customizer] The function to customize cloning. - * @returns {*} Returns the deep cloned value. - * @see _.cloneWith - * @example - * - * function customizer(value) { - * if (_.isElement(value)) { - * return value.cloneNode(true); - * } - * } - * - * var el = _.cloneDeepWith(document.body, customizer); - * - * console.log(el === document.body); - * // => false - * console.log(el.nodeName); - * // => 'BODY' - * console.log(el.childNodes.length); - * // => 20 - */ - function cloneDeepWith(value, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return baseClone(value, CLONE_DEEP_FLAG | CLONE_SYMBOLS_FLAG, customizer); - } - - /** - * Checks if `object` conforms to `source` by invoking the predicate - * properties of `source` with the corresponding property values of `object`. - * - * **Note:** This method is equivalent to `_.conforms` when `source` is - * partially applied. - * - * @static - * @memberOf _ - * @since 4.14.0 - * @category Lang - * @param {Object} object The object to inspect. - * @param {Object} source The object of property predicates to conform to. - * @returns {boolean} Returns `true` if `object` conforms, else `false`. - * @example - * - * var object = { 'a': 1, 'b': 2 }; - * - * _.conformsTo(object, { 'b': function(n) { return n > 1; } }); - * // => true - * - * _.conformsTo(object, { 'b': function(n) { return n > 2; } }); - * // => false - */ - function conformsTo(object, source) { - return source == null || baseConformsTo(object, source, keys(source)); - } - - /** - * Performs a - * [`SameValueZero`](http://ecma-international.org/ecma-262/7.0/#sec-samevaluezero) - * comparison between two values to determine if they are equivalent. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. - * @example - * - * var object = { 'a': 1 }; - * var other = { 'a': 1 }; - * - * _.eq(object, object); - * // => true - * - * _.eq(object, other); - * // => false - * - * _.eq('a', 'a'); - * // => true - * - * _.eq('a', Object('a')); - * // => false - * - * _.eq(NaN, NaN); - * // => true - */ - function eq(value, other) { - return value === other || (value !== value && other !== other); - } - - /** - * Checks if `value` is greater than `other`. - * - * @static - * @memberOf _ - * @since 3.9.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is greater than `other`, - * else `false`. - * @see _.lt - * @example - * - * _.gt(3, 1); - * // => true - * - * _.gt(3, 3); - * // => false - * - * _.gt(1, 3); - * // => false - */ - var gt = createRelationalOperation(baseGt); - - /** - * Checks if `value` is greater than or equal to `other`. - * - * @static - * @memberOf _ - * @since 3.9.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is greater than or equal to - * `other`, else `false`. - * @see _.lte - * @example - * - * _.gte(3, 1); - * // => true - * - * _.gte(3, 3); - * // => true - * - * _.gte(1, 3); - * // => false - */ - var gte = createRelationalOperation(function(value, other) { - return value >= other; - }); - - /** - * Checks if `value` is likely an `arguments` object. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an `arguments` object, - * else `false`. - * @example - * - * _.isArguments(function() { return arguments; }()); - * // => true - * - * _.isArguments([1, 2, 3]); - * // => false - */ - var isArguments = baseIsArguments(function() { return arguments; }()) ? baseIsArguments : function(value) { - return isObjectLike(value) && hasOwnProperty.call(value, 'callee') && - !propertyIsEnumerable.call(value, 'callee'); - }; - - /** - * Checks if `value` is classified as an `Array` object. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array, else `false`. - * @example - * - * _.isArray([1, 2, 3]); - * // => true - * - * _.isArray(document.body.children); - * // => false - * - * _.isArray('abc'); - * // => false - * - * _.isArray(_.noop); - * // => false - */ - var isArray = Array.isArray; - - /** - * Checks if `value` is classified as an `ArrayBuffer` object. - * - * @static - * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array buffer, else `false`. - * @example - * - * _.isArrayBuffer(new ArrayBuffer(2)); - * // => true - * - * _.isArrayBuffer(new Array(2)); - * // => false - */ - var isArrayBuffer = nodeIsArrayBuffer ? baseUnary(nodeIsArrayBuffer) : baseIsArrayBuffer; - - /** - * Checks if `value` is array-like. A value is considered array-like if it's - * not a function and has a `value.length` that's an integer greater than or - * equal to `0` and less than or equal to `Number.MAX_SAFE_INTEGER`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is array-like, else `false`. - * @example - * - * _.isArrayLike([1, 2, 3]); - * // => true - * - * _.isArrayLike(document.body.children); - * // => true - * - * _.isArrayLike('abc'); - * // => true - * - * _.isArrayLike(_.noop); - * // => false - */ - function isArrayLike(value) { - return value != null && isLength(value.length) && !isFunction(value); - } - - /** - * This method is like `_.isArrayLike` except that it also checks if `value` - * is an object. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an array-like object, - * else `false`. - * @example - * - * _.isArrayLikeObject([1, 2, 3]); - * // => true - * - * _.isArrayLikeObject(document.body.children); - * // => true - * - * _.isArrayLikeObject('abc'); - * // => false - * - * _.isArrayLikeObject(_.noop); - * // => false - */ - function isArrayLikeObject(value) { - return isObjectLike(value) && isArrayLike(value); - } - - /** - * Checks if `value` is classified as a boolean primitive or object. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a boolean, else `false`. - * @example - * - * _.isBoolean(false); - * // => true - * - * _.isBoolean(null); - * // => false - */ - function isBoolean(value) { - return value === true || value === false || - (isObjectLike(value) && baseGetTag(value) == boolTag); - } - - /** - * Checks if `value` is a buffer. - * - * @static - * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a buffer, else `false`. - * @example - * - * _.isBuffer(new Buffer(2)); - * // => true - * - * _.isBuffer(new Uint8Array(2)); - * // => false - */ - var isBuffer = nativeIsBuffer || stubFalse; - - /** - * Checks if `value` is classified as a `Date` object. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a date object, else `false`. - * @example - * - * _.isDate(new Date); - * // => true - * - * _.isDate('Mon April 23 2012'); - * // => false - */ - var isDate = nodeIsDate ? baseUnary(nodeIsDate) : baseIsDate; - - /** - * Checks if `value` is likely a DOM element. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a DOM element, else `false`. - * @example - * - * _.isElement(document.body); - * // => true - * - * _.isElement(''); - * // => false - */ - function isElement(value) { - return isObjectLike(value) && value.nodeType === 1 && !isPlainObject(value); - } - - /** - * Checks if `value` is an empty object, collection, map, or set. - * - * Objects are considered empty if they have no own enumerable string keyed - * properties. - * - * Array-like values such as `arguments` objects, arrays, buffers, strings, or - * jQuery-like collections are considered empty if they have a `length` of `0`. - * Similarly, maps and sets are considered empty if they have a `size` of `0`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is empty, else `false`. - * @example - * - * _.isEmpty(null); - * // => true - * - * _.isEmpty(true); - * // => true - * - * _.isEmpty(1); - * // => true - * - * _.isEmpty([1, 2, 3]); - * // => false - * - * _.isEmpty({ 'a': 1 }); - * // => false - */ - function isEmpty(value) { - if (value == null) { - return true; - } - if (isArrayLike(value) && - (isArray(value) || typeof value == 'string' || typeof value.splice == 'function' || - isBuffer(value) || isTypedArray(value) || isArguments(value))) { - return !value.length; - } - var tag = getTag(value); - if (tag == mapTag || tag == setTag) { - return !value.size; - } - if (isPrototype(value)) { - return !baseKeys(value).length; - } - for (var key in value) { - if (hasOwnProperty.call(value, key)) { - return false; - } - } - return true; - } - - /** - * Performs a deep comparison between two values to determine if they are - * equivalent. - * - * **Note:** This method supports comparing arrays, array buffers, booleans, - * date objects, error objects, maps, numbers, `Object` objects, regexes, - * sets, strings, symbols, and typed arrays. `Object` objects are compared - * by their own, not inherited, enumerable properties. Functions and DOM - * nodes are compared by strict equality, i.e. `===`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. - * @example - * - * var object = { 'a': 1 }; - * var other = { 'a': 1 }; - * - * _.isEqual(object, other); - * // => true - * - * object === other; - * // => false - */ - function isEqual(value, other) { - return baseIsEqual(value, other); - } - - /** - * This method is like `_.isEqual` except that it accepts `customizer` which - * is invoked to compare values. If `customizer` returns `undefined`, comparisons - * are handled by the method instead. The `customizer` is invoked with up to - * six arguments: (objValue, othValue [, index|key, object, other, stack]). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @param {Function} [customizer] The function to customize comparisons. - * @returns {boolean} Returns `true` if the values are equivalent, else `false`. - * @example - * - * function isGreeting(value) { - * return /^h(?:i|ello)$/.test(value); - * } - * - * function customizer(objValue, othValue) { - * if (isGreeting(objValue) && isGreeting(othValue)) { - * return true; - * } - * } - * - * var array = ['hello', 'goodbye']; - * var other = ['hi', 'goodbye']; - * - * _.isEqualWith(array, other, customizer); - * // => true - */ - function isEqualWith(value, other, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - var result = customizer ? customizer(value, other) : undefined; - return result === undefined ? baseIsEqual(value, other, undefined, customizer) : !!result; - } - - /** - * Checks if `value` is an `Error`, `EvalError`, `RangeError`, `ReferenceError`, - * `SyntaxError`, `TypeError`, or `URIError` object. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an error object, else `false`. - * @example - * - * _.isError(new Error); - * // => true - * - * _.isError(Error); - * // => false - */ - function isError(value) { - if (!isObjectLike(value)) { - return false; - } - var tag = baseGetTag(value); - return tag == errorTag || tag == domExcTag || - (typeof value.message == 'string' && typeof value.name == 'string' && !isPlainObject(value)); - } - - /** - * Checks if `value` is a finite primitive number. - * - * **Note:** This method is based on - * [`Number.isFinite`](https://mdn.io/Number/isFinite). - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a finite number, else `false`. - * @example - * - * _.isFinite(3); - * // => true - * - * _.isFinite(Number.MIN_VALUE); - * // => true - * - * _.isFinite(Infinity); - * // => false - * - * _.isFinite('3'); - * // => false - */ - function isFinite(value) { - return typeof value == 'number' && nativeIsFinite(value); - } - - /** - * Checks if `value` is classified as a `Function` object. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a function, else `false`. - * @example - * - * _.isFunction(_); - * // => true - * - * _.isFunction(/abc/); - * // => false - */ - function isFunction(value) { - if (!isObject(value)) { - return false; - } - // The use of `Object#toString` avoids issues with the `typeof` operator - // in Safari 9 which returns 'object' for typed arrays and other constructors. - var tag = baseGetTag(value); - return tag == funcTag || tag == genTag || tag == asyncTag || tag == proxyTag; - } - - /** - * Checks if `value` is an integer. - * - * **Note:** This method is based on - * [`Number.isInteger`](https://mdn.io/Number/isInteger). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an integer, else `false`. - * @example - * - * _.isInteger(3); - * // => true - * - * _.isInteger(Number.MIN_VALUE); - * // => false - * - * _.isInteger(Infinity); - * // => false - * - * _.isInteger('3'); - * // => false - */ - function isInteger(value) { - return typeof value == 'number' && value == toInteger(value); - } - - /** - * Checks if `value` is a valid array-like length. - * - * **Note:** This method is loosely based on - * [`ToLength`](http://ecma-international.org/ecma-262/7.0/#sec-tolength). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a valid length, else `false`. - * @example - * - * _.isLength(3); - * // => true - * - * _.isLength(Number.MIN_VALUE); - * // => false - * - * _.isLength(Infinity); - * // => false - * - * _.isLength('3'); - * // => false - */ - function isLength(value) { - return typeof value == 'number' && - value > -1 && value % 1 == 0 && value <= MAX_SAFE_INTEGER; - } - - /** - * Checks if `value` is the - * [language type](http://www.ecma-international.org/ecma-262/7.0/#sec-ecmascript-language-types) - * of `Object`. (e.g. arrays, functions, objects, regexes, `new Number(0)`, and `new String('')`) - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is an object, else `false`. - * @example - * - * _.isObject({}); - * // => true - * - * _.isObject([1, 2, 3]); - * // => true - * - * _.isObject(_.noop); - * // => true - * - * _.isObject(null); - * // => false - */ - function isObject(value) { - var type = typeof value; - return value != null && (type == 'object' || type == 'function'); - } - - /** - * Checks if `value` is object-like. A value is object-like if it's not `null` - * and has a `typeof` result of "object". - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is object-like, else `false`. - * @example - * - * _.isObjectLike({}); - * // => true - * - * _.isObjectLike([1, 2, 3]); - * // => true - * - * _.isObjectLike(_.noop); - * // => false - * - * _.isObjectLike(null); - * // => false - */ - function isObjectLike(value) { - return value != null && typeof value == 'object'; - } - - /** - * Checks if `value` is classified as a `Map` object. - * - * @static - * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a map, else `false`. - * @example - * - * _.isMap(new Map); - * // => true - * - * _.isMap(new WeakMap); - * // => false - */ - var isMap = nodeIsMap ? baseUnary(nodeIsMap) : baseIsMap; - - /** - * Performs a partial deep comparison between `object` and `source` to - * determine if `object` contains equivalent property values. - * - * **Note:** This method is equivalent to `_.matches` when `source` is - * partially applied. - * - * Partial comparisons will match empty array and empty object `source` - * values against any array or object value, respectively. See `_.isEqual` - * for a list of supported value comparisons. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {Object} object The object to inspect. - * @param {Object} source The object of property values to match. - * @returns {boolean} Returns `true` if `object` is a match, else `false`. - * @example - * - * var object = { 'a': 1, 'b': 2 }; - * - * _.isMatch(object, { 'b': 2 }); - * // => true - * - * _.isMatch(object, { 'b': 1 }); - * // => false - */ - function isMatch(object, source) { - return object === source || baseIsMatch(object, source, getMatchData(source)); - } - - /** - * This method is like `_.isMatch` except that it accepts `customizer` which - * is invoked to compare values. If `customizer` returns `undefined`, comparisons - * are handled by the method instead. The `customizer` is invoked with five - * arguments: (objValue, srcValue, index|key, object, source). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {Object} object The object to inspect. - * @param {Object} source The object of property values to match. - * @param {Function} [customizer] The function to customize comparisons. - * @returns {boolean} Returns `true` if `object` is a match, else `false`. - * @example - * - * function isGreeting(value) { - * return /^h(?:i|ello)$/.test(value); - * } - * - * function customizer(objValue, srcValue) { - * if (isGreeting(objValue) && isGreeting(srcValue)) { - * return true; - * } - * } - * - * var object = { 'greeting': 'hello' }; - * var source = { 'greeting': 'hi' }; - * - * _.isMatchWith(object, source, customizer); - * // => true - */ - function isMatchWith(object, source, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return baseIsMatch(object, source, getMatchData(source), customizer); - } - - /** - * Checks if `value` is `NaN`. - * - * **Note:** This method is based on - * [`Number.isNaN`](https://mdn.io/Number/isNaN) and is not the same as - * global [`isNaN`](https://mdn.io/isNaN) which returns `true` for - * `undefined` and other non-number values. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `NaN`, else `false`. - * @example - * - * _.isNaN(NaN); - * // => true - * - * _.isNaN(new Number(NaN)); - * // => true - * - * isNaN(undefined); - * // => true - * - * _.isNaN(undefined); - * // => false - */ - function isNaN(value) { - // An `NaN` primitive is the only value that is not equal to itself. - // Perform the `toStringTag` check first to avoid errors with some - // ActiveX objects in IE. - return isNumber(value) && value != +value; - } - - /** - * Checks if `value` is a pristine native function. - * - * **Note:** This method can't reliably detect native functions in the presence - * of the core-js package because core-js circumvents this kind of detection. - * Despite multiple requests, the core-js maintainer has made it clear: any - * attempt to fix the detection will be obstructed. As a result, we're left - * with little choice but to throw an error. Unfortunately, this also affects - * packages, like [babel-polyfill](https://www.npmjs.com/package/babel-polyfill), - * which rely on core-js. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a native function, - * else `false`. - * @example - * - * _.isNative(Array.prototype.push); - * // => true - * - * _.isNative(_); - * // => false - */ - function isNative(value) { - if (isMaskable(value)) { - throw new Error(CORE_ERROR_TEXT); - } - return baseIsNative(value); - } - - /** - * Checks if `value` is `null`. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `null`, else `false`. - * @example - * - * _.isNull(null); - * // => true - * - * _.isNull(void 0); - * // => false - */ - function isNull(value) { - return value === null; - } - - /** - * Checks if `value` is `null` or `undefined`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is nullish, else `false`. - * @example - * - * _.isNil(null); - * // => true - * - * _.isNil(void 0); - * // => true - * - * _.isNil(NaN); - * // => false - */ - function isNil(value) { - return value == null; - } - - /** - * Checks if `value` is classified as a `Number` primitive or object. - * - * **Note:** To exclude `Infinity`, `-Infinity`, and `NaN`, which are - * classified as numbers, use the `_.isFinite` method. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a number, else `false`. - * @example - * - * _.isNumber(3); - * // => true - * - * _.isNumber(Number.MIN_VALUE); - * // => true - * - * _.isNumber(Infinity); - * // => true - * - * _.isNumber('3'); - * // => false - */ - function isNumber(value) { - return typeof value == 'number' || - (isObjectLike(value) && baseGetTag(value) == numberTag); - } - - /** - * Checks if `value` is a plain object, that is, an object created by the - * `Object` constructor or one with a `[[Prototype]]` of `null`. - * - * @static - * @memberOf _ - * @since 0.8.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a plain object, else `false`. - * @example - * - * function Foo() { - * this.a = 1; - * } - * - * _.isPlainObject(new Foo); - * // => false - * - * _.isPlainObject([1, 2, 3]); - * // => false - * - * _.isPlainObject({ 'x': 0, 'y': 0 }); - * // => true - * - * _.isPlainObject(Object.create(null)); - * // => true - */ - function isPlainObject(value) { - if (!isObjectLike(value) || baseGetTag(value) != objectTag) { - return false; - } - var proto = getPrototype(value); - if (proto === null) { - return true; - } - var Ctor = hasOwnProperty.call(proto, 'constructor') && proto.constructor; - return typeof Ctor == 'function' && Ctor instanceof Ctor && - funcToString.call(Ctor) == objectCtorString; - } - - /** - * Checks if `value` is classified as a `RegExp` object. - * - * @static - * @memberOf _ - * @since 0.1.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a regexp, else `false`. - * @example - * - * _.isRegExp(/abc/); - * // => true - * - * _.isRegExp('/abc/'); - * // => false - */ - var isRegExp = nodeIsRegExp ? baseUnary(nodeIsRegExp) : baseIsRegExp; - - /** - * Checks if `value` is a safe integer. An integer is safe if it's an IEEE-754 - * double precision number which isn't the result of a rounded unsafe integer. - * - * **Note:** This method is based on - * [`Number.isSafeInteger`](https://mdn.io/Number/isSafeInteger). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a safe integer, else `false`. - * @example - * - * _.isSafeInteger(3); - * // => true - * - * _.isSafeInteger(Number.MIN_VALUE); - * // => false - * - * _.isSafeInteger(Infinity); - * // => false - * - * _.isSafeInteger('3'); - * // => false - */ - function isSafeInteger(value) { - return isInteger(value) && value >= -MAX_SAFE_INTEGER && value <= MAX_SAFE_INTEGER; - } - - /** - * Checks if `value` is classified as a `Set` object. - * - * @static - * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a set, else `false`. - * @example - * - * _.isSet(new Set); - * // => true - * - * _.isSet(new WeakSet); - * // => false - */ - var isSet = nodeIsSet ? baseUnary(nodeIsSet) : baseIsSet; - - /** - * Checks if `value` is classified as a `String` primitive or object. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a string, else `false`. - * @example - * - * _.isString('abc'); - * // => true - * - * _.isString(1); - * // => false - */ - function isString(value) { - return typeof value == 'string' || - (!isArray(value) && isObjectLike(value) && baseGetTag(value) == stringTag); - } - - /** - * Checks if `value` is classified as a `Symbol` primitive or object. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a symbol, else `false`. - * @example - * - * _.isSymbol(Symbol.iterator); - * // => true - * - * _.isSymbol('abc'); - * // => false - */ - function isSymbol(value) { - return typeof value == 'symbol' || - (isObjectLike(value) && baseGetTag(value) == symbolTag); - } - - /** - * Checks if `value` is classified as a typed array. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a typed array, else `false`. - * @example - * - * _.isTypedArray(new Uint8Array); - * // => true - * - * _.isTypedArray([]); - * // => false - */ - var isTypedArray = nodeIsTypedArray ? baseUnary(nodeIsTypedArray) : baseIsTypedArray; - - /** - * Checks if `value` is `undefined`. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is `undefined`, else `false`. - * @example - * - * _.isUndefined(void 0); - * // => true - * - * _.isUndefined(null); - * // => false - */ - function isUndefined(value) { - return value === undefined; - } - - /** - * Checks if `value` is classified as a `WeakMap` object. - * - * @static - * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a weak map, else `false`. - * @example - * - * _.isWeakMap(new WeakMap); - * // => true - * - * _.isWeakMap(new Map); - * // => false - */ - function isWeakMap(value) { - return isObjectLike(value) && getTag(value) == weakMapTag; - } - - /** - * Checks if `value` is classified as a `WeakSet` object. - * - * @static - * @memberOf _ - * @since 4.3.0 - * @category Lang - * @param {*} value The value to check. - * @returns {boolean} Returns `true` if `value` is a weak set, else `false`. - * @example - * - * _.isWeakSet(new WeakSet); - * // => true - * - * _.isWeakSet(new Set); - * // => false - */ - function isWeakSet(value) { - return isObjectLike(value) && baseGetTag(value) == weakSetTag; - } - - /** - * Checks if `value` is less than `other`. - * - * @static - * @memberOf _ - * @since 3.9.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is less than `other`, - * else `false`. - * @see _.gt - * @example - * - * _.lt(1, 3); - * // => true - * - * _.lt(3, 3); - * // => false - * - * _.lt(3, 1); - * // => false - */ - var lt = createRelationalOperation(baseLt); - - /** - * Checks if `value` is less than or equal to `other`. - * - * @static - * @memberOf _ - * @since 3.9.0 - * @category Lang - * @param {*} value The value to compare. - * @param {*} other The other value to compare. - * @returns {boolean} Returns `true` if `value` is less than or equal to - * `other`, else `false`. - * @see _.gte - * @example - * - * _.lte(1, 3); - * // => true - * - * _.lte(3, 3); - * // => true - * - * _.lte(3, 1); - * // => false - */ - var lte = createRelationalOperation(function(value, other) { - return value <= other; - }); - - /** - * Converts `value` to an array. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Lang - * @param {*} value The value to convert. - * @returns {Array} Returns the converted array. - * @example - * - * _.toArray({ 'a': 1, 'b': 2 }); - * // => [1, 2] - * - * _.toArray('abc'); - * // => ['a', 'b', 'c'] - * - * _.toArray(1); - * // => [] - * - * _.toArray(null); - * // => [] - */ - function toArray(value) { - if (!value) { - return []; - } - if (isArrayLike(value)) { - return isString(value) ? stringToArray(value) : copyArray(value); - } - if (symIterator && value[symIterator]) { - return iteratorToArray(value[symIterator]()); - } - var tag = getTag(value), - func = tag == mapTag ? mapToArray : (tag == setTag ? setToArray : values); - - return func(value); - } - - /** - * Converts `value` to a finite number. - * - * @static - * @memberOf _ - * @since 4.12.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted number. - * @example - * - * _.toFinite(3.2); - * // => 3.2 - * - * _.toFinite(Number.MIN_VALUE); - * // => 5e-324 - * - * _.toFinite(Infinity); - * // => 1.7976931348623157e+308 - * - * _.toFinite('3.2'); - * // => 3.2 - */ - function toFinite(value) { - if (!value) { - return value === 0 ? value : 0; - } - value = toNumber(value); - if (value === INFINITY || value === -INFINITY) { - var sign = (value < 0 ? -1 : 1); - return sign * MAX_INTEGER; - } - return value === value ? value : 0; - } - - /** - * Converts `value` to an integer. - * - * **Note:** This method is loosely based on - * [`ToInteger`](http://www.ecma-international.org/ecma-262/7.0/#sec-tointeger). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted integer. - * @example - * - * _.toInteger(3.2); - * // => 3 - * - * _.toInteger(Number.MIN_VALUE); - * // => 0 - * - * _.toInteger(Infinity); - * // => 1.7976931348623157e+308 - * - * _.toInteger('3.2'); - * // => 3 - */ - function toInteger(value) { - var result = toFinite(value), - remainder = result % 1; - - return result === result ? (remainder ? result - remainder : result) : 0; - } - - /** - * Converts `value` to an integer suitable for use as the length of an - * array-like object. - * - * **Note:** This method is based on - * [`ToLength`](http://ecma-international.org/ecma-262/7.0/#sec-tolength). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted integer. - * @example - * - * _.toLength(3.2); - * // => 3 - * - * _.toLength(Number.MIN_VALUE); - * // => 0 - * - * _.toLength(Infinity); - * // => 4294967295 - * - * _.toLength('3.2'); - * // => 3 - */ - function toLength(value) { - return value ? baseClamp(toInteger(value), 0, MAX_ARRAY_LENGTH) : 0; - } - - /** - * Converts `value` to a number. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to process. - * @returns {number} Returns the number. - * @example - * - * _.toNumber(3.2); - * // => 3.2 - * - * _.toNumber(Number.MIN_VALUE); - * // => 5e-324 - * - * _.toNumber(Infinity); - * // => Infinity - * - * _.toNumber('3.2'); - * // => 3.2 - */ - function toNumber(value) { - if (typeof value == 'number') { - return value; - } - if (isSymbol(value)) { - return NAN; - } - if (isObject(value)) { - var other = typeof value.valueOf == 'function' ? value.valueOf() : value; - value = isObject(other) ? (other + '') : other; - } - if (typeof value != 'string') { - return value === 0 ? value : +value; - } - value = value.replace(reTrim, ''); - var isBinary = reIsBinary.test(value); - return (isBinary || reIsOctal.test(value)) - ? freeParseInt(value.slice(2), isBinary ? 2 : 8) - : (reIsBadHex.test(value) ? NAN : +value); - } - - /** - * Converts `value` to a plain object flattening inherited enumerable string - * keyed properties of `value` to own properties of the plain object. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {Object} Returns the converted plain object. - * @example - * - * function Foo() { - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.assign({ 'a': 1 }, new Foo); - * // => { 'a': 1, 'b': 2 } - * - * _.assign({ 'a': 1 }, _.toPlainObject(new Foo)); - * // => { 'a': 1, 'b': 2, 'c': 3 } - */ - function toPlainObject(value) { - return copyObject(value, keysIn(value)); - } - - /** - * Converts `value` to a safe integer. A safe integer can be compared and - * represented correctly. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {number} Returns the converted integer. - * @example - * - * _.toSafeInteger(3.2); - * // => 3 - * - * _.toSafeInteger(Number.MIN_VALUE); - * // => 0 - * - * _.toSafeInteger(Infinity); - * // => 9007199254740991 - * - * _.toSafeInteger('3.2'); - * // => 3 - */ - function toSafeInteger(value) { - return value - ? baseClamp(toInteger(value), -MAX_SAFE_INTEGER, MAX_SAFE_INTEGER) - : (value === 0 ? value : 0); - } - - /** - * Converts `value` to a string. An empty string is returned for `null` - * and `undefined` values. The sign of `-0` is preserved. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Lang - * @param {*} value The value to convert. - * @returns {string} Returns the converted string. - * @example - * - * _.toString(null); - * // => '' - * - * _.toString(-0); - * // => '-0' - * - * _.toString([1, 2, 3]); - * // => '1,2,3' - */ - function toString(value) { - return value == null ? '' : baseToString(value); - } - - /*------------------------------------------------------------------------*/ - - /** - * Assigns own enumerable string keyed properties of source objects to the - * destination object. Source objects are applied from left to right. - * Subsequent sources overwrite property assignments of previous sources. - * - * **Note:** This method mutates `object` and is loosely based on - * [`Object.assign`](https://mdn.io/Object/assign). - * - * @static - * @memberOf _ - * @since 0.10.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.assignIn - * @example - * - * function Foo() { - * this.a = 1; - * } - * - * function Bar() { - * this.c = 3; - * } - * - * Foo.prototype.b = 2; - * Bar.prototype.d = 4; - * - * _.assign({ 'a': 0 }, new Foo, new Bar); - * // => { 'a': 1, 'c': 3 } - */ - var assign = createAssigner(function(object, source) { - if (isPrototype(source) || isArrayLike(source)) { - copyObject(source, keys(source), object); - return; - } - for (var key in source) { - if (hasOwnProperty.call(source, key)) { - assignValue(object, key, source[key]); - } - } - }); - - /** - * This method is like `_.assign` except that it iterates over own and - * inherited source properties. - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @alias extend - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.assign - * @example - * - * function Foo() { - * this.a = 1; - * } - * - * function Bar() { - * this.c = 3; - * } - * - * Foo.prototype.b = 2; - * Bar.prototype.d = 4; - * - * _.assignIn({ 'a': 0 }, new Foo, new Bar); - * // => { 'a': 1, 'b': 2, 'c': 3, 'd': 4 } - */ - var assignIn = createAssigner(function(object, source) { - copyObject(source, keysIn(source), object); - }); - - /** - * This method is like `_.assignIn` except that it accepts `customizer` - * which is invoked to produce the assigned values. If `customizer` returns - * `undefined`, assignment is handled by the method instead. The `customizer` - * is invoked with five arguments: (objValue, srcValue, key, object, source). - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @alias extendWith - * @category Object - * @param {Object} object The destination object. - * @param {...Object} sources The source objects. - * @param {Function} [customizer] The function to customize assigned values. - * @returns {Object} Returns `object`. - * @see _.assignWith - * @example - * - * function customizer(objValue, srcValue) { - * return _.isUndefined(objValue) ? srcValue : objValue; - * } - * - * var defaults = _.partialRight(_.assignInWith, customizer); - * - * defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); - * // => { 'a': 1, 'b': 2 } - */ - var assignInWith = createAssigner(function(object, source, srcIndex, customizer) { - copyObject(source, keysIn(source), object, customizer); - }); - - /** - * This method is like `_.assign` except that it accepts `customizer` - * which is invoked to produce the assigned values. If `customizer` returns - * `undefined`, assignment is handled by the method instead. The `customizer` - * is invoked with five arguments: (objValue, srcValue, key, object, source). - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} sources The source objects. - * @param {Function} [customizer] The function to customize assigned values. - * @returns {Object} Returns `object`. - * @see _.assignInWith - * @example - * - * function customizer(objValue, srcValue) { - * return _.isUndefined(objValue) ? srcValue : objValue; - * } - * - * var defaults = _.partialRight(_.assignWith, customizer); - * - * defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); - * // => { 'a': 1, 'b': 2 } - */ - var assignWith = createAssigner(function(object, source, srcIndex, customizer) { - copyObject(source, keys(source), object, customizer); - }); - - /** - * Creates an array of values corresponding to `paths` of `object`. - * - * @static - * @memberOf _ - * @since 1.0.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {...(string|string[])} [paths] The property paths to pick. - * @returns {Array} Returns the picked values. - * @example - * - * var object = { 'a': [{ 'b': { 'c': 3 } }, 4] }; - * - * _.at(object, ['a[0].b.c', 'a[1]']); - * // => [3, 4] - */ - var at = flatRest(baseAt); - - /** - * Creates an object that inherits from the `prototype` object. If a - * `properties` object is given, its own enumerable string keyed properties - * are assigned to the created object. - * - * @static - * @memberOf _ - * @since 2.3.0 - * @category Object - * @param {Object} prototype The object to inherit from. - * @param {Object} [properties] The properties to assign to the object. - * @returns {Object} Returns the new object. - * @example - * - * function Shape() { - * this.x = 0; - * this.y = 0; - * } - * - * function Circle() { - * Shape.call(this); - * } - * - * Circle.prototype = _.create(Shape.prototype, { - * 'constructor': Circle - * }); - * - * var circle = new Circle; - * circle instanceof Circle; - * // => true - * - * circle instanceof Shape; - * // => true - */ - function create(prototype, properties) { - var result = baseCreate(prototype); - return properties == null ? result : baseAssign(result, properties); - } - - /** - * Assigns own and inherited enumerable string keyed properties of source - * objects to the destination object for all destination properties that - * resolve to `undefined`. Source objects are applied from left to right. - * Once a property is set, additional values of the same property are ignored. - * - * **Note:** This method mutates `object`. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.defaultsDeep - * @example - * - * _.defaults({ 'a': 1 }, { 'b': 2 }, { 'a': 3 }); - * // => { 'a': 1, 'b': 2 } - */ - var defaults = baseRest(function(object, sources) { - object = Object(object); - - var index = -1; - var length = sources.length; - var guard = length > 2 ? sources[2] : undefined; - - if (guard && isIterateeCall(sources[0], sources[1], guard)) { - length = 1; - } - - while (++index < length) { - var source = sources[index]; - var props = keysIn(source); - var propsIndex = -1; - var propsLength = props.length; - - while (++propsIndex < propsLength) { - var key = props[propsIndex]; - var value = object[key]; - - if (value === undefined || - (eq(value, objectProto[key]) && !hasOwnProperty.call(object, key))) { - object[key] = source[key]; - } - } - } - - return object; - }); - - /** - * This method is like `_.defaults` except that it recursively assigns - * default properties. - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 3.10.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @see _.defaults - * @example - * - * _.defaultsDeep({ 'a': { 'b': 2 } }, { 'a': { 'b': 1, 'c': 3 } }); - * // => { 'a': { 'b': 2, 'c': 3 } } - */ - var defaultsDeep = baseRest(function(args) { - args.push(undefined, customDefaultsMerge); - return apply(mergeWith, undefined, args); - }); - - /** - * This method is like `_.find` except that it returns the key of the first - * element `predicate` returns truthy for instead of the element itself. - * - * @static - * @memberOf _ - * @since 1.1.0 - * @category Object - * @param {Object} object The object to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {string|undefined} Returns the key of the matched element, - * else `undefined`. - * @example - * - * var users = { - * 'barney': { 'age': 36, 'active': true }, - * 'fred': { 'age': 40, 'active': false }, - * 'pebbles': { 'age': 1, 'active': true } - * }; - * - * _.findKey(users, function(o) { return o.age < 40; }); - * // => 'barney' (iteration order is not guaranteed) - * - * // The `_.matches` iteratee shorthand. - * _.findKey(users, { 'age': 1, 'active': true }); - * // => 'pebbles' - * - * // The `_.matchesProperty` iteratee shorthand. - * _.findKey(users, ['active', false]); - * // => 'fred' - * - * // The `_.property` iteratee shorthand. - * _.findKey(users, 'active'); - * // => 'barney' - */ - function findKey(object, predicate) { - return baseFindKey(object, getIteratee(predicate, 3), baseForOwn); - } - - /** - * This method is like `_.findKey` except that it iterates over elements of - * a collection in the opposite order. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Object - * @param {Object} object The object to inspect. - * @param {Function} [predicate=_.identity] The function invoked per iteration. - * @returns {string|undefined} Returns the key of the matched element, - * else `undefined`. - * @example - * - * var users = { - * 'barney': { 'age': 36, 'active': true }, - * 'fred': { 'age': 40, 'active': false }, - * 'pebbles': { 'age': 1, 'active': true } - * }; - * - * _.findLastKey(users, function(o) { return o.age < 40; }); - * // => returns 'pebbles' assuming `_.findKey` returns 'barney' - * - * // The `_.matches` iteratee shorthand. - * _.findLastKey(users, { 'age': 36, 'active': true }); - * // => 'barney' - * - * // The `_.matchesProperty` iteratee shorthand. - * _.findLastKey(users, ['active', false]); - * // => 'fred' - * - * // The `_.property` iteratee shorthand. - * _.findLastKey(users, 'active'); - * // => 'pebbles' - */ - function findLastKey(object, predicate) { - return baseFindKey(object, getIteratee(predicate, 3), baseForOwnRight); - } - - /** - * Iterates over own and inherited enumerable string keyed properties of an - * object and invokes `iteratee` for each property. The iteratee is invoked - * with three arguments: (value, key, object). Iteratee functions may exit - * iteration early by explicitly returning `false`. - * - * @static - * @memberOf _ - * @since 0.3.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forInRight - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.forIn(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'a', 'b', then 'c' (iteration order is not guaranteed). - */ - function forIn(object, iteratee) { - return object == null - ? object - : baseFor(object, getIteratee(iteratee, 3), keysIn); - } - - /** - * This method is like `_.forIn` except that it iterates over properties of - * `object` in the opposite order. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forIn - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.forInRight(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'c', 'b', then 'a' assuming `_.forIn` logs 'a', 'b', then 'c'. - */ - function forInRight(object, iteratee) { - return object == null - ? object - : baseForRight(object, getIteratee(iteratee, 3), keysIn); - } - - /** - * Iterates over own enumerable string keyed properties of an object and - * invokes `iteratee` for each property. The iteratee is invoked with three - * arguments: (value, key, object). Iteratee functions may exit iteration - * early by explicitly returning `false`. - * - * @static - * @memberOf _ - * @since 0.3.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forOwnRight - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.forOwn(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'a' then 'b' (iteration order is not guaranteed). - */ - function forOwn(object, iteratee) { - return object && baseForOwn(object, getIteratee(iteratee, 3)); - } - - /** - * This method is like `_.forOwn` except that it iterates over properties of - * `object` in the opposite order. - * - * @static - * @memberOf _ - * @since 2.0.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns `object`. - * @see _.forOwn - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.forOwnRight(new Foo, function(value, key) { - * console.log(key); - * }); - * // => Logs 'b' then 'a' assuming `_.forOwn` logs 'a' then 'b'. - */ - function forOwnRight(object, iteratee) { - return object && baseForOwnRight(object, getIteratee(iteratee, 3)); - } - - /** - * Creates an array of function property names from own enumerable properties - * of `object`. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The object to inspect. - * @returns {Array} Returns the function names. - * @see _.functionsIn - * @example - * - * function Foo() { - * this.a = _.constant('a'); - * this.b = _.constant('b'); - * } - * - * Foo.prototype.c = _.constant('c'); - * - * _.functions(new Foo); - * // => ['a', 'b'] - */ - function functions(object) { - return object == null ? [] : baseFunctions(object, keys(object)); - } - - /** - * Creates an array of function property names from own and inherited - * enumerable properties of `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The object to inspect. - * @returns {Array} Returns the function names. - * @see _.functions - * @example - * - * function Foo() { - * this.a = _.constant('a'); - * this.b = _.constant('b'); - * } - * - * Foo.prototype.c = _.constant('c'); - * - * _.functionsIn(new Foo); - * // => ['a', 'b', 'c'] - */ - function functionsIn(object) { - return object == null ? [] : baseFunctions(object, keysIn(object)); - } - - /** - * Gets the value at `path` of `object`. If the resolved value is - * `undefined`, the `defaultValue` is returned in its place. - * - * @static - * @memberOf _ - * @since 3.7.0 - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path of the property to get. - * @param {*} [defaultValue] The value returned for `undefined` resolved values. - * @returns {*} Returns the resolved value. - * @example - * - * var object = { 'a': [{ 'b': { 'c': 3 } }] }; - * - * _.get(object, 'a[0].b.c'); - * // => 3 - * - * _.get(object, ['a', '0', 'b', 'c']); - * // => 3 - * - * _.get(object, 'a.b.c', 'default'); - * // => 'default' - */ - function get(object, path, defaultValue) { - var result = object == null ? undefined : baseGet(object, path); - return result === undefined ? defaultValue : result; - } - - /** - * Checks if `path` is a direct property of `object`. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path to check. - * @returns {boolean} Returns `true` if `path` exists, else `false`. - * @example - * - * var object = { 'a': { 'b': 2 } }; - * var other = _.create({ 'a': _.create({ 'b': 2 }) }); - * - * _.has(object, 'a'); - * // => true - * - * _.has(object, 'a.b'); - * // => true - * - * _.has(object, ['a', 'b']); - * // => true - * - * _.has(other, 'a'); - * // => false - */ - function has(object, path) { - return object != null && hasPath(object, path, baseHas); - } - - /** - * Checks if `path` is a direct or inherited property of `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path to check. - * @returns {boolean} Returns `true` if `path` exists, else `false`. - * @example - * - * var object = _.create({ 'a': _.create({ 'b': 2 }) }); - * - * _.hasIn(object, 'a'); - * // => true - * - * _.hasIn(object, 'a.b'); - * // => true - * - * _.hasIn(object, ['a', 'b']); - * // => true - * - * _.hasIn(object, 'b'); - * // => false - */ - function hasIn(object, path) { - return object != null && hasPath(object, path, baseHasIn); - } - - /** - * Creates an object composed of the inverted keys and values of `object`. - * If `object` contains duplicate values, subsequent values overwrite - * property assignments of previous values. - * - * @static - * @memberOf _ - * @since 0.7.0 - * @category Object - * @param {Object} object The object to invert. - * @returns {Object} Returns the new inverted object. - * @example - * - * var object = { 'a': 1, 'b': 2, 'c': 1 }; - * - * _.invert(object); - * // => { '1': 'c', '2': 'b' } - */ - var invert = createInverter(function(result, value, key) { - if (value != null && - typeof value.toString != 'function') { - value = nativeObjectToString.call(value); - } - - result[value] = key; - }, constant(identity)); - - /** - * This method is like `_.invert` except that the inverted object is generated - * from the results of running each element of `object` thru `iteratee`. The - * corresponding inverted value of each inverted key is an array of keys - * responsible for generating the inverted value. The iteratee is invoked - * with one argument: (value). - * - * @static - * @memberOf _ - * @since 4.1.0 - * @category Object - * @param {Object} object The object to invert. - * @param {Function} [iteratee=_.identity] The iteratee invoked per element. - * @returns {Object} Returns the new inverted object. - * @example - * - * var object = { 'a': 1, 'b': 2, 'c': 1 }; - * - * _.invertBy(object); - * // => { '1': ['a', 'c'], '2': ['b'] } - * - * _.invertBy(object, function(value) { - * return 'group' + value; - * }); - * // => { 'group1': ['a', 'c'], 'group2': ['b'] } - */ - var invertBy = createInverter(function(result, value, key) { - if (value != null && - typeof value.toString != 'function') { - value = nativeObjectToString.call(value); - } - - if (hasOwnProperty.call(result, value)) { - result[value].push(key); - } else { - result[value] = [key]; - } - }, getIteratee); - - /** - * Invokes the method at `path` of `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path of the method to invoke. - * @param {...*} [args] The arguments to invoke the method with. - * @returns {*} Returns the result of the invoked method. - * @example - * - * var object = { 'a': [{ 'b': { 'c': [1, 2, 3, 4] } }] }; - * - * _.invoke(object, 'a[0].b.c.slice', 1, 3); - * // => [2, 3] - */ - var invoke = baseRest(baseInvoke); - - /** - * Creates an array of the own enumerable property names of `object`. - * - * **Note:** Non-object values are coerced to objects. See the - * [ES spec](http://ecma-international.org/ecma-262/7.0/#sec-object.keys) - * for more details. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.keys(new Foo); - * // => ['a', 'b'] (iteration order is not guaranteed) - * - * _.keys('hi'); - * // => ['0', '1'] - */ - function keys(object) { - return isArrayLike(object) ? arrayLikeKeys(object) : baseKeys(object); - } - - /** - * Creates an array of the own and inherited enumerable property names of `object`. - * - * **Note:** Non-object values are coerced to objects. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Object - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property names. - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.keysIn(new Foo); - * // => ['a', 'b', 'c'] (iteration order is not guaranteed) - */ - function keysIn(object) { - return isArrayLike(object) ? arrayLikeKeys(object, true) : baseKeysIn(object); - } - - /** - * The opposite of `_.mapValues`; this method creates an object with the - * same values as `object` and keys generated by running each own enumerable - * string keyed property of `object` thru `iteratee`. The iteratee is invoked - * with three arguments: (value, key, object). - * - * @static - * @memberOf _ - * @since 3.8.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns the new mapped object. - * @see _.mapValues - * @example - * - * _.mapKeys({ 'a': 1, 'b': 2 }, function(value, key) { - * return key + value; - * }); - * // => { 'a1': 1, 'b2': 2 } - */ - function mapKeys(object, iteratee) { - var result = {}; - iteratee = getIteratee(iteratee, 3); - - baseForOwn(object, function(value, key, object) { - baseAssignValue(result, iteratee(value, key, object), value); - }); - return result; - } - - /** - * Creates an object with the same keys as `object` and values generated - * by running each own enumerable string keyed property of `object` thru - * `iteratee`. The iteratee is invoked with three arguments: - * (value, key, object). - * - * @static - * @memberOf _ - * @since 2.4.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @returns {Object} Returns the new mapped object. - * @see _.mapKeys - * @example - * - * var users = { - * 'fred': { 'user': 'fred', 'age': 40 }, - * 'pebbles': { 'user': 'pebbles', 'age': 1 } - * }; - * - * _.mapValues(users, function(o) { return o.age; }); - * // => { 'fred': 40, 'pebbles': 1 } (iteration order is not guaranteed) - * - * // The `_.property` iteratee shorthand. - * _.mapValues(users, 'age'); - * // => { 'fred': 40, 'pebbles': 1 } (iteration order is not guaranteed) - */ - function mapValues(object, iteratee) { - var result = {}; - iteratee = getIteratee(iteratee, 3); - - baseForOwn(object, function(value, key, object) { - baseAssignValue(result, key, iteratee(value, key, object)); - }); - return result; - } - - /** - * This method is like `_.assign` except that it recursively merges own and - * inherited enumerable string keyed properties of source objects into the - * destination object. Source properties that resolve to `undefined` are - * skipped if a destination value exists. Array and plain object properties - * are merged recursively. Other objects and value types are overridden by - * assignment. Source objects are applied from left to right. Subsequent - * sources overwrite property assignments of previous sources. - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 0.5.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} [sources] The source objects. - * @returns {Object} Returns `object`. - * @example - * - * var object = { - * 'a': [{ 'b': 2 }, { 'd': 4 }] - * }; - * - * var other = { - * 'a': [{ 'c': 3 }, { 'e': 5 }] - * }; - * - * _.merge(object, other); - * // => { 'a': [{ 'b': 2, 'c': 3 }, { 'd': 4, 'e': 5 }] } - */ - var merge = createAssigner(function(object, source, srcIndex) { - baseMerge(object, source, srcIndex); - }); - - /** - * This method is like `_.merge` except that it accepts `customizer` which - * is invoked to produce the merged values of the destination and source - * properties. If `customizer` returns `undefined`, merging is handled by the - * method instead. The `customizer` is invoked with six arguments: - * (objValue, srcValue, key, object, source, stack). - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The destination object. - * @param {...Object} sources The source objects. - * @param {Function} customizer The function to customize assigned values. - * @returns {Object} Returns `object`. - * @example - * - * function customizer(objValue, srcValue) { - * if (_.isArray(objValue)) { - * return objValue.concat(srcValue); - * } - * } - * - * var object = { 'a': [1], 'b': [2] }; - * var other = { 'a': [3], 'b': [4] }; - * - * _.mergeWith(object, other, customizer); - * // => { 'a': [1, 3], 'b': [2, 4] } - */ - var mergeWith = createAssigner(function(object, source, srcIndex, customizer) { - baseMerge(object, source, srcIndex, customizer); - }); - - /** - * The opposite of `_.pick`; this method creates an object composed of the - * own and inherited enumerable property paths of `object` that are not omitted. - * - * **Note:** This method is considerably slower than `_.pick`. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The source object. - * @param {...(string|string[])} [paths] The property paths to omit. - * @returns {Object} Returns the new object. - * @example - * - * var object = { 'a': 1, 'b': '2', 'c': 3 }; - * - * _.omit(object, ['a', 'c']); - * // => { 'b': '2' } - */ - var omit = flatRest(function(object, paths) { - var result = {}; - if (object == null) { - return result; - } - var isDeep = false; - paths = arrayMap(paths, function(path) { - path = castPath(path, object); - isDeep || (isDeep = path.length > 1); - return path; - }); - copyObject(object, getAllKeysIn(object), result); - if (isDeep) { - result = baseClone(result, CLONE_DEEP_FLAG | CLONE_FLAT_FLAG | CLONE_SYMBOLS_FLAG, customOmitClone); - } - var length = paths.length; - while (length--) { - baseUnset(result, paths[length]); - } - return result; - }); - - /** - * The opposite of `_.pickBy`; this method creates an object composed of - * the own and inherited enumerable string keyed properties of `object` that - * `predicate` doesn't return truthy for. The predicate is invoked with two - * arguments: (value, key). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The source object. - * @param {Function} [predicate=_.identity] The function invoked per property. - * @returns {Object} Returns the new object. - * @example - * - * var object = { 'a': 1, 'b': '2', 'c': 3 }; - * - * _.omitBy(object, _.isNumber); - * // => { 'b': '2' } - */ - function omitBy(object, predicate) { - return pickBy(object, negate(getIteratee(predicate))); - } - - /** - * Creates an object composed of the picked `object` properties. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The source object. - * @param {...(string|string[])} [paths] The property paths to pick. - * @returns {Object} Returns the new object. - * @example - * - * var object = { 'a': 1, 'b': '2', 'c': 3 }; - * - * _.pick(object, ['a', 'c']); - * // => { 'a': 1, 'c': 3 } - */ - var pick = flatRest(function(object, paths) { - return object == null ? {} : basePick(object, paths); - }); - - /** - * Creates an object composed of the `object` properties `predicate` returns - * truthy for. The predicate is invoked with two arguments: (value, key). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The source object. - * @param {Function} [predicate=_.identity] The function invoked per property. - * @returns {Object} Returns the new object. - * @example - * - * var object = { 'a': 1, 'b': '2', 'c': 3 }; - * - * _.pickBy(object, _.isNumber); - * // => { 'a': 1, 'c': 3 } - */ - function pickBy(object, predicate) { - if (object == null) { - return {}; - } - var props = arrayMap(getAllKeysIn(object), function(prop) { - return [prop]; - }); - predicate = getIteratee(predicate); - return basePickBy(object, props, function(value, path) { - return predicate(value, path[0]); - }); - } - - /** - * This method is like `_.get` except that if the resolved value is a - * function it's invoked with the `this` binding of its parent object and - * its result is returned. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The object to query. - * @param {Array|string} path The path of the property to resolve. - * @param {*} [defaultValue] The value returned for `undefined` resolved values. - * @returns {*} Returns the resolved value. - * @example - * - * var object = { 'a': [{ 'b': { 'c1': 3, 'c2': _.constant(4) } }] }; - * - * _.result(object, 'a[0].b.c1'); - * // => 3 - * - * _.result(object, 'a[0].b.c2'); - * // => 4 - * - * _.result(object, 'a[0].b.c3', 'default'); - * // => 'default' - * - * _.result(object, 'a[0].b.c3', _.constant('default')); - * // => 'default' - */ - function result(object, path, defaultValue) { - path = castPath(path, object); - - var index = -1, - length = path.length; - - // Ensure the loop is entered when path is empty. - if (!length) { - length = 1; - object = undefined; - } - while (++index < length) { - var value = object == null ? undefined : object[toKey(path[index])]; - if (value === undefined) { - index = length; - value = defaultValue; - } - object = isFunction(value) ? value.call(object) : value; - } - return object; - } - - /** - * Sets the value at `path` of `object`. If a portion of `path` doesn't exist, - * it's created. Arrays are created for missing index properties while objects - * are created for all other missing properties. Use `_.setWith` to customize - * `path` creation. - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 3.7.0 - * @category Object - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to set. - * @param {*} value The value to set. - * @returns {Object} Returns `object`. - * @example - * - * var object = { 'a': [{ 'b': { 'c': 3 } }] }; - * - * _.set(object, 'a[0].b.c', 4); - * console.log(object.a[0].b.c); - * // => 4 - * - * _.set(object, ['x', '0', 'y', 'z'], 5); - * console.log(object.x[0].y.z); - * // => 5 - */ - function set(object, path, value) { - return object == null ? object : baseSet(object, path, value); - } - - /** - * This method is like `_.set` except that it accepts `customizer` which is - * invoked to produce the objects of `path`. If `customizer` returns `undefined` - * path creation is handled by the method instead. The `customizer` is invoked - * with three arguments: (nsValue, key, nsObject). - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to set. - * @param {*} value The value to set. - * @param {Function} [customizer] The function to customize assigned values. - * @returns {Object} Returns `object`. - * @example - * - * var object = {}; - * - * _.setWith(object, '[0][1]', 'a', Object); - * // => { '0': { '1': 'a' } } - */ - function setWith(object, path, value, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return object == null ? object : baseSet(object, path, value, customizer); - } - - /** - * Creates an array of own enumerable string keyed-value pairs for `object` - * which can be consumed by `_.fromPairs`. If `object` is a map or set, its - * entries are returned. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @alias entries - * @category Object - * @param {Object} object The object to query. - * @returns {Array} Returns the key-value pairs. - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.toPairs(new Foo); - * // => [['a', 1], ['b', 2]] (iteration order is not guaranteed) - */ - var toPairs = createToPairs(keys); - - /** - * Creates an array of own and inherited enumerable string keyed-value pairs - * for `object` which can be consumed by `_.fromPairs`. If `object` is a map - * or set, its entries are returned. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @alias entriesIn - * @category Object - * @param {Object} object The object to query. - * @returns {Array} Returns the key-value pairs. - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.toPairsIn(new Foo); - * // => [['a', 1], ['b', 2], ['c', 3]] (iteration order is not guaranteed) - */ - var toPairsIn = createToPairs(keysIn); - - /** - * An alternative to `_.reduce`; this method transforms `object` to a new - * `accumulator` object which is the result of running each of its own - * enumerable string keyed properties thru `iteratee`, with each invocation - * potentially mutating the `accumulator` object. If `accumulator` is not - * provided, a new object with the same `[[Prototype]]` will be used. The - * iteratee is invoked with four arguments: (accumulator, value, key, object). - * Iteratee functions may exit iteration early by explicitly returning `false`. - * - * @static - * @memberOf _ - * @since 1.3.0 - * @category Object - * @param {Object} object The object to iterate over. - * @param {Function} [iteratee=_.identity] The function invoked per iteration. - * @param {*} [accumulator] The custom accumulator value. - * @returns {*} Returns the accumulated value. - * @example - * - * _.transform([2, 3, 4], function(result, n) { - * result.push(n *= n); - * return n % 2 == 0; - * }, []); - * // => [4, 9] - * - * _.transform({ 'a': 1, 'b': 2, 'c': 1 }, function(result, value, key) { - * (result[value] || (result[value] = [])).push(key); - * }, {}); - * // => { '1': ['a', 'c'], '2': ['b'] } - */ - function transform(object, iteratee, accumulator) { - var isArr = isArray(object), - isArrLike = isArr || isBuffer(object) || isTypedArray(object); - - iteratee = getIteratee(iteratee, 4); - if (accumulator == null) { - var Ctor = object && object.constructor; - if (isArrLike) { - accumulator = isArr ? new Ctor : []; - } - else if (isObject(object)) { - accumulator = isFunction(Ctor) ? baseCreate(getPrototype(object)) : {}; - } - else { - accumulator = {}; - } - } - (isArrLike ? arrayEach : baseForOwn)(object, function(value, index, object) { - return iteratee(accumulator, value, index, object); - }); - return accumulator; - } - - /** - * Removes the property at `path` of `object`. - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Object - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to unset. - * @returns {boolean} Returns `true` if the property is deleted, else `false`. - * @example - * - * var object = { 'a': [{ 'b': { 'c': 7 } }] }; - * _.unset(object, 'a[0].b.c'); - * // => true - * - * console.log(object); - * // => { 'a': [{ 'b': {} }] }; - * - * _.unset(object, ['a', '0', 'b', 'c']); - * // => true - * - * console.log(object); - * // => { 'a': [{ 'b': {} }] }; - */ - function unset(object, path) { - return object == null ? true : baseUnset(object, path); - } - - /** - * This method is like `_.set` except that accepts `updater` to produce the - * value to set. Use `_.updateWith` to customize `path` creation. The `updater` - * is invoked with one argument: (value). - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.6.0 - * @category Object - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to set. - * @param {Function} updater The function to produce the updated value. - * @returns {Object} Returns `object`. - * @example - * - * var object = { 'a': [{ 'b': { 'c': 3 } }] }; - * - * _.update(object, 'a[0].b.c', function(n) { return n * n; }); - * console.log(object.a[0].b.c); - * // => 9 - * - * _.update(object, 'x[0].y.z', function(n) { return n ? n + 1 : 0; }); - * console.log(object.x[0].y.z); - * // => 0 - */ - function update(object, path, updater) { - return object == null ? object : baseUpdate(object, path, castFunction(updater)); - } - - /** - * This method is like `_.update` except that it accepts `customizer` which is - * invoked to produce the objects of `path`. If `customizer` returns `undefined` - * path creation is handled by the method instead. The `customizer` is invoked - * with three arguments: (nsValue, key, nsObject). - * - * **Note:** This method mutates `object`. - * - * @static - * @memberOf _ - * @since 4.6.0 - * @category Object - * @param {Object} object The object to modify. - * @param {Array|string} path The path of the property to set. - * @param {Function} updater The function to produce the updated value. - * @param {Function} [customizer] The function to customize assigned values. - * @returns {Object} Returns `object`. - * @example - * - * var object = {}; - * - * _.updateWith(object, '[0][1]', _.constant('a'), Object); - * // => { '0': { '1': 'a' } } - */ - function updateWith(object, path, updater, customizer) { - customizer = typeof customizer == 'function' ? customizer : undefined; - return object == null ? object : baseUpdate(object, path, castFunction(updater), customizer); - } - - /** - * Creates an array of the own enumerable string keyed property values of `object`. - * - * **Note:** Non-object values are coerced to objects. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category Object - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property values. - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.values(new Foo); - * // => [1, 2] (iteration order is not guaranteed) - * - * _.values('hi'); - * // => ['h', 'i'] - */ - function values(object) { - return object == null ? [] : baseValues(object, keys(object)); - } - - /** - * Creates an array of the own and inherited enumerable string keyed property - * values of `object`. - * - * **Note:** Non-object values are coerced to objects. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category Object - * @param {Object} object The object to query. - * @returns {Array} Returns the array of property values. - * @example - * - * function Foo() { - * this.a = 1; - * this.b = 2; - * } - * - * Foo.prototype.c = 3; - * - * _.valuesIn(new Foo); - * // => [1, 2, 3] (iteration order is not guaranteed) - */ - function valuesIn(object) { - return object == null ? [] : baseValues(object, keysIn(object)); - } - - /*------------------------------------------------------------------------*/ - - /** - * Clamps `number` within the inclusive `lower` and `upper` bounds. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category Number - * @param {number} number The number to clamp. - * @param {number} [lower] The lower bound. - * @param {number} upper The upper bound. - * @returns {number} Returns the clamped number. - * @example - * - * _.clamp(-10, -5, 5); - * // => -5 - * - * _.clamp(10, -5, 5); - * // => 5 - */ - function clamp(number, lower, upper) { - if (upper === undefined) { - upper = lower; - lower = undefined; - } - if (upper !== undefined) { - upper = toNumber(upper); - upper = upper === upper ? upper : 0; - } - if (lower !== undefined) { - lower = toNumber(lower); - lower = lower === lower ? lower : 0; - } - return baseClamp(toNumber(number), lower, upper); - } - - /** - * Checks if `n` is between `start` and up to, but not including, `end`. If - * `end` is not specified, it's set to `start` with `start` then set to `0`. - * If `start` is greater than `end` the params are swapped to support - * negative ranges. - * - * @static - * @memberOf _ - * @since 3.3.0 - * @category Number - * @param {number} number The number to check. - * @param {number} [start=0] The start of the range. - * @param {number} end The end of the range. - * @returns {boolean} Returns `true` if `number` is in the range, else `false`. - * @see _.range, _.rangeRight - * @example - * - * _.inRange(3, 2, 4); - * // => true - * - * _.inRange(4, 8); - * // => true - * - * _.inRange(4, 2); - * // => false - * - * _.inRange(2, 2); - * // => false - * - * _.inRange(1.2, 2); - * // => true - * - * _.inRange(5.2, 4); - * // => false - * - * _.inRange(-3, -2, -6); - * // => true - */ - function inRange(number, start, end) { - start = toFinite(start); - if (end === undefined) { - end = start; - start = 0; - } else { - end = toFinite(end); - } - number = toNumber(number); - return baseInRange(number, start, end); - } - - /** - * Produces a random number between the inclusive `lower` and `upper` bounds. - * If only one argument is provided a number between `0` and the given number - * is returned. If `floating` is `true`, or either `lower` or `upper` are - * floats, a floating-point number is returned instead of an integer. - * - * **Note:** JavaScript follows the IEEE-754 standard for resolving - * floating-point values which can produce unexpected results. - * - * @static - * @memberOf _ - * @since 0.7.0 - * @category Number - * @param {number} [lower=0] The lower bound. - * @param {number} [upper=1] The upper bound. - * @param {boolean} [floating] Specify returning a floating-point number. - * @returns {number} Returns the random number. - * @example - * - * _.random(0, 5); - * // => an integer between 0 and 5 - * - * _.random(5); - * // => also an integer between 0 and 5 - * - * _.random(5, true); - * // => a floating-point number between 0 and 5 - * - * _.random(1.2, 5.2); - * // => a floating-point number between 1.2 and 5.2 - */ - function random(lower, upper, floating) { - if (floating && typeof floating != 'boolean' && isIterateeCall(lower, upper, floating)) { - upper = floating = undefined; - } - if (floating === undefined) { - if (typeof upper == 'boolean') { - floating = upper; - upper = undefined; - } - else if (typeof lower == 'boolean') { - floating = lower; - lower = undefined; - } - } - if (lower === undefined && upper === undefined) { - lower = 0; - upper = 1; - } - else { - lower = toFinite(lower); - if (upper === undefined) { - upper = lower; - lower = 0; - } else { - upper = toFinite(upper); - } - } - if (lower > upper) { - var temp = lower; - lower = upper; - upper = temp; - } - if (floating || lower % 1 || upper % 1) { - var rand = nativeRandom(); - return nativeMin(lower + (rand * (upper - lower + freeParseFloat('1e-' + ((rand + '').length - 1)))), upper); - } - return baseRandom(lower, upper); - } - - /*------------------------------------------------------------------------*/ - - /** - * Converts `string` to [camel case](https://en.wikipedia.org/wiki/CamelCase). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to convert. - * @returns {string} Returns the camel cased string. - * @example - * - * _.camelCase('Foo Bar'); - * // => 'fooBar' - * - * _.camelCase('--foo-bar--'); - * // => 'fooBar' - * - * _.camelCase('__FOO_BAR__'); - * // => 'fooBar' - */ - var camelCase = createCompounder(function(result, word, index) { - word = word.toLowerCase(); - return result + (index ? capitalize(word) : word); - }); - - /** - * Converts the first character of `string` to upper case and the remaining - * to lower case. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to capitalize. - * @returns {string} Returns the capitalized string. - * @example - * - * _.capitalize('FRED'); - * // => 'Fred' - */ - function capitalize(string) { - return upperFirst(toString(string).toLowerCase()); - } - - /** - * Deburrs `string` by converting - * [Latin-1 Supplement](https://en.wikipedia.org/wiki/Latin-1_Supplement_(Unicode_block)#Character_table) - * and [Latin Extended-A](https://en.wikipedia.org/wiki/Latin_Extended-A) - * letters to basic Latin letters and removing - * [combining diacritical marks](https://en.wikipedia.org/wiki/Combining_Diacritical_Marks). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to deburr. - * @returns {string} Returns the deburred string. - * @example - * - * _.deburr('déjà vu'); - * // => 'deja vu' - */ - function deburr(string) { - string = toString(string); - return string && string.replace(reLatin, deburrLetter).replace(reComboMark, ''); - } - - /** - * Checks if `string` ends with the given target string. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to inspect. - * @param {string} [target] The string to search for. - * @param {number} [position=string.length] The position to search up to. - * @returns {boolean} Returns `true` if `string` ends with `target`, - * else `false`. - * @example - * - * _.endsWith('abc', 'c'); - * // => true - * - * _.endsWith('abc', 'b'); - * // => false - * - * _.endsWith('abc', 'b', 2); - * // => true - */ - function endsWith(string, target, position) { - string = toString(string); - target = baseToString(target); - - var length = string.length; - position = position === undefined - ? length - : baseClamp(toInteger(position), 0, length); - - var end = position; - position -= target.length; - return position >= 0 && string.slice(position, end) == target; - } - - /** - * Converts the characters "&", "<", ">", '"', and "'" in `string` to their - * corresponding HTML entities. - * - * **Note:** No other characters are escaped. To escape additional - * characters use a third-party library like [_he_](https://mths.be/he). - * - * Though the ">" character is escaped for symmetry, characters like - * ">" and "/" don't need escaping in HTML and have no special meaning - * unless they're part of a tag or unquoted attribute value. See - * [Mathias Bynens's article](https://mathiasbynens.be/notes/ambiguous-ampersands) - * (under "semi-related fun fact") for more details. - * - * When working with HTML you should always - * [quote attribute values](http://wonko.com/post/html-escaping) to reduce - * XSS vectors. - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category String - * @param {string} [string=''] The string to escape. - * @returns {string} Returns the escaped string. - * @example - * - * _.escape('fred, barney, & pebbles'); - * // => 'fred, barney, & pebbles' - */ - function escape(string) { - string = toString(string); - return (string && reHasUnescapedHtml.test(string)) - ? string.replace(reUnescapedHtml, escapeHtmlChar) - : string; - } - - /** - * Escapes the `RegExp` special characters "^", "$", "\", ".", "*", "+", - * "?", "(", ")", "[", "]", "{", "}", and "|" in `string`. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to escape. - * @returns {string} Returns the escaped string. - * @example - * - * _.escapeRegExp('[lodash](https://lodash.com/)'); - * // => '\[lodash\]\(https://lodash\.com/\)' - */ - function escapeRegExp(string) { - string = toString(string); - return (string && reHasRegExpChar.test(string)) - ? string.replace(reRegExpChar, '\\$&') - : string; - } - - /** - * Converts `string` to - * [kebab case](https://en.wikipedia.org/wiki/Letter_case#Special_case_styles). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to convert. - * @returns {string} Returns the kebab cased string. - * @example - * - * _.kebabCase('Foo Bar'); - * // => 'foo-bar' - * - * _.kebabCase('fooBar'); - * // => 'foo-bar' - * - * _.kebabCase('__FOO_BAR__'); - * // => 'foo-bar' - */ - var kebabCase = createCompounder(function(result, word, index) { - return result + (index ? '-' : '') + word.toLowerCase(); - }); - - /** - * Converts `string`, as space separated words, to lower case. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category String - * @param {string} [string=''] The string to convert. - * @returns {string} Returns the lower cased string. - * @example - * - * _.lowerCase('--Foo-Bar--'); - * // => 'foo bar' - * - * _.lowerCase('fooBar'); - * // => 'foo bar' - * - * _.lowerCase('__FOO_BAR__'); - * // => 'foo bar' - */ - var lowerCase = createCompounder(function(result, word, index) { - return result + (index ? ' ' : '') + word.toLowerCase(); - }); - - /** - * Converts the first character of `string` to lower case. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category String - * @param {string} [string=''] The string to convert. - * @returns {string} Returns the converted string. - * @example - * - * _.lowerFirst('Fred'); - * // => 'fred' - * - * _.lowerFirst('FRED'); - * // => 'fRED' - */ - var lowerFirst = createCaseFirst('toLowerCase'); - - /** - * Pads `string` on the left and right sides if it's shorter than `length`. - * Padding characters are truncated if they can't be evenly divided by `length`. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to pad. - * @param {number} [length=0] The padding length. - * @param {string} [chars=' '] The string used as padding. - * @returns {string} Returns the padded string. - * @example - * - * _.pad('abc', 8); - * // => ' abc ' - * - * _.pad('abc', 8, '_-'); - * // => '_-abc_-_' - * - * _.pad('abc', 3); - * // => 'abc' - */ - function pad(string, length, chars) { - string = toString(string); - length = toInteger(length); - - var strLength = length ? stringSize(string) : 0; - if (!length || strLength >= length) { - return string; - } - var mid = (length - strLength) / 2; - return ( - createPadding(nativeFloor(mid), chars) + - string + - createPadding(nativeCeil(mid), chars) - ); - } - - /** - * Pads `string` on the right side if it's shorter than `length`. Padding - * characters are truncated if they exceed `length`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category String - * @param {string} [string=''] The string to pad. - * @param {number} [length=0] The padding length. - * @param {string} [chars=' '] The string used as padding. - * @returns {string} Returns the padded string. - * @example - * - * _.padEnd('abc', 6); - * // => 'abc ' - * - * _.padEnd('abc', 6, '_-'); - * // => 'abc_-_' - * - * _.padEnd('abc', 3); - * // => 'abc' - */ - function padEnd(string, length, chars) { - string = toString(string); - length = toInteger(length); - - var strLength = length ? stringSize(string) : 0; - return (length && strLength < length) - ? (string + createPadding(length - strLength, chars)) - : string; - } - - /** - * Pads `string` on the left side if it's shorter than `length`. Padding - * characters are truncated if they exceed `length`. - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category String - * @param {string} [string=''] The string to pad. - * @param {number} [length=0] The padding length. - * @param {string} [chars=' '] The string used as padding. - * @returns {string} Returns the padded string. - * @example - * - * _.padStart('abc', 6); - * // => ' abc' - * - * _.padStart('abc', 6, '_-'); - * // => '_-_abc' - * - * _.padStart('abc', 3); - * // => 'abc' - */ - function padStart(string, length, chars) { - string = toString(string); - length = toInteger(length); - - var strLength = length ? stringSize(string) : 0; - return (length && strLength < length) - ? (createPadding(length - strLength, chars) + string) - : string; - } - - /** - * Converts `string` to an integer of the specified radix. If `radix` is - * `undefined` or `0`, a `radix` of `10` is used unless `value` is a - * hexadecimal, in which case a `radix` of `16` is used. - * - * **Note:** This method aligns with the - * [ES5 implementation](https://es5.github.io/#x15.1.2.2) of `parseInt`. - * - * @static - * @memberOf _ - * @since 1.1.0 - * @category String - * @param {string} string The string to convert. - * @param {number} [radix=10] The radix to interpret `value` by. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {number} Returns the converted integer. - * @example - * - * _.parseInt('08'); - * // => 8 - * - * _.map(['6', '08', '10'], _.parseInt); - * // => [6, 8, 10] - */ - function parseInt(string, radix, guard) { - if (guard || radix == null) { - radix = 0; - } else if (radix) { - radix = +radix; - } - return nativeParseInt(toString(string).replace(reTrimStart, ''), radix || 0); - } - - /** - * Repeats the given string `n` times. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to repeat. - * @param {number} [n=1] The number of times to repeat the string. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {string} Returns the repeated string. - * @example - * - * _.repeat('*', 3); - * // => '***' - * - * _.repeat('abc', 2); - * // => 'abcabc' - * - * _.repeat('abc', 0); - * // => '' - */ - function repeat(string, n, guard) { - if ((guard ? isIterateeCall(string, n, guard) : n === undefined)) { - n = 1; - } else { - n = toInteger(n); - } - return baseRepeat(toString(string), n); - } - - /** - * Replaces matches for `pattern` in `string` with `replacement`. - * - * **Note:** This method is based on - * [`String#replace`](https://mdn.io/String/replace). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category String - * @param {string} [string=''] The string to modify. - * @param {RegExp|string} pattern The pattern to replace. - * @param {Function|string} replacement The match replacement. - * @returns {string} Returns the modified string. - * @example - * - * _.replace('Hi Fred', 'Fred', 'Barney'); - * // => 'Hi Barney' - */ - function replace() { - var args = arguments, - string = toString(args[0]); - - return args.length < 3 ? string : string.replace(args[1], args[2]); - } - - /** - * Converts `string` to - * [snake case](https://en.wikipedia.org/wiki/Snake_case). - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to convert. - * @returns {string} Returns the snake cased string. - * @example - * - * _.snakeCase('Foo Bar'); - * // => 'foo_bar' - * - * _.snakeCase('fooBar'); - * // => 'foo_bar' - * - * _.snakeCase('--FOO-BAR--'); - * // => 'foo_bar' - */ - var snakeCase = createCompounder(function(result, word, index) { - return result + (index ? '_' : '') + word.toLowerCase(); - }); - - /** - * Splits `string` by `separator`. - * - * **Note:** This method is based on - * [`String#split`](https://mdn.io/String/split). - * - * @static - * @memberOf _ - * @since 4.0.0 - * @category String - * @param {string} [string=''] The string to split. - * @param {RegExp|string} separator The separator pattern to split by. - * @param {number} [limit] The length to truncate results to. - * @returns {Array} Returns the string segments. - * @example - * - * _.split('a-b-c', '-', 2); - * // => ['a', 'b'] - */ - function split(string, separator, limit) { - if (limit && typeof limit != 'number' && isIterateeCall(string, separator, limit)) { - separator = limit = undefined; - } - limit = limit === undefined ? MAX_ARRAY_LENGTH : limit >>> 0; - if (!limit) { - return []; - } - string = toString(string); - if (string && ( - typeof separator == 'string' || - (separator != null && !isRegExp(separator)) - )) { - separator = baseToString(separator); - if (!separator && hasUnicode(string)) { - return castSlice(stringToArray(string), 0, limit); - } - } - return string.split(separator, limit); - } - - /** - * Converts `string` to - * [start case](https://en.wikipedia.org/wiki/Letter_case#Stylistic_or_specialised_usage). - * - * @static - * @memberOf _ - * @since 3.1.0 - * @category String - * @param {string} [string=''] The string to convert. - * @returns {string} Returns the start cased string. - * @example - * - * _.startCase('--foo-bar--'); - * // => 'Foo Bar' - * - * _.startCase('fooBar'); - * // => 'Foo Bar' - * - * _.startCase('__FOO_BAR__'); - * // => 'FOO BAR' - */ - var startCase = createCompounder(function(result, word, index) { - return result + (index ? ' ' : '') + upperFirst(word); - }); - - /** - * Checks if `string` starts with the given target string. - * - * @static - * @memberOf _ - * @since 3.0.0 - * @category String - * @param {string} [string=''] The string to inspect. - * @param {string} [target] The string to search for. - * @param {number} [position=0] The position to search from. - * @returns {boolean} Returns `true` if `string` starts with `target`, - * else `false`. - * @example - * - * _.startsWith('abc', 'a'); - * // => true - * - * _.startsWith('abc', 'b'); - * // => false - * - * _.startsWith('abc', 'b', 1); - * // => true - */ - function startsWith(string, target, position) { - string = toString(string); - position = position == null - ? 0 - : baseClamp(toInteger(position), 0, string.length); - - target = baseToString(target); - return string.slice(position, position + target.length) == target; - } - - /** - * Creates a compiled template function that can interpolate data properties - * in "interpolate" delimiters, HTML-escape interpolated data properties in - * "escape" delimiters, and execute JavaScript in "evaluate" delimiters. Data - * properties may be accessed as free variables in the template. If a setting - * object is given, it takes precedence over `_.templateSettings` values. - * - * **Note:** In the development build `_.template` utilizes - * [sourceURLs](http://www.html5rocks.com/en/tutorials/developertools/sourcemaps/#toc-sourceurl) - * for easier debugging. - * - * For more information on precompiling templates see - * [lodash's custom builds documentation](https://lodash.com/custom-builds). - * - * For more information on Chrome extension sandboxes see - * [Chrome's extensions documentation](https://developer.chrome.com/extensions/sandboxingEval). - * - * @static - * @since 0.1.0 - * @memberOf _ - * @category String - * @param {string} [string=''] The template string. - * @param {Object} [options={}] The options object. - * @param {RegExp} [options.escape=_.templateSettings.escape] - * The HTML "escape" delimiter. - * @param {RegExp} [options.evaluate=_.templateSettings.evaluate] - * The "evaluate" delimiter. - * @param {Object} [options.imports=_.templateSettings.imports] - * An object to import into the template as free variables. - * @param {RegExp} [options.interpolate=_.templateSettings.interpolate] - * The "interpolate" delimiter. - * @param {string} [options.sourceURL='lodash.templateSources[n]'] - * The sourceURL of the compiled template. - * @param {string} [options.variable='obj'] - * The data object variable name. - * @param- {Object} [guard] Enables use as an iteratee for methods like `_.map`. - * @returns {Function} Returns the compiled template function. - * @example - * - * // Use the "interpolate" delimiter to create a compiled template. - * var compiled = _.template('hello <%= user %>!'); - * compiled({ 'user': 'fred' }); - * // => 'hello fred!' - * - * // Use the HTML "escape" delimiter to escape data property values. - * var compiled = _.template('<%- value %>'); - * compiled({ 'value': '