Use centralized scripts with S3 interactions

.github/workflows/rebuild-all.yaml

@@ -9,6 +9,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Repository Dispatch
-        run: ./ingest/vendored/trigger monkeypox rebuild
+        run: ./ingest/vendored/trigger nextstrain/monkeypox rebuild
         env:
           PAT_GITHUB_DISPATCH: ${{ secrets.GH_TOKEN_NEXTSTRAIN_BOT_WORKFLOW_DISPATCH }}

ingest/vendored/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/nextstrain/ingest
 	branch = main
-	commit = 5d908187d13cbce27253af5163b4803d6bac03a6
-	parent = 79e690f465cfafc68157c18b499daf10c36b648d
+	commit = 1eb8b30428d5f66adac201f0a246a7ab4bdc9792
+	parent = a7ccb51fecede6336ef06ec2a980db27aa5821b4
 	method = merge
 	cmdver = 0.4.6

ingest/vendored/trigger

@@ -3,7 +3,7 @@ set -euo pipefail
 
 : "${PAT_GITHUB_DISPATCH:=}"
 
-repo="${1:?A repository name is required as the first argument.}"
+github_repo="${1:?A GitHub repository with owner and repository name is required as the first argument.}"
 event_type="${2:?An event type is required as the second argument.}"
 shift 2
 
@@ -12,7 +12,7 @@ if [[ $# -eq 0 && -z $PAT_GITHUB_DISPATCH ]]; then
 You must specify options to curl for your GitHub credentials.  For example, you
 can specify your GitHub username, and will be prompted for your password:
 
-  $0 $repo $event_type --user <your-github-username>
+  $0 $github_repo $event_type --user <your-github-username>
 
 Be sure to enter a personal access token¹ as your password since GitHub has
 discontinued password authentication to the API starting on November 13, 2020².
@@ -26,7 +26,7 @@ file³:
 
 and then tell curl to use it:
 
-  $0 $repo $event_type --netrc
+  $0 $github_repo $event_type --netrc
 
 which will then not require you to type your password every time.
 
@@ -42,7 +42,7 @@ if [[ -n $PAT_GITHUB_DISPATCH ]]; then
   auth="Authorization: Bearer ${PAT_GITHUB_DISPATCH}"
 fi
 
-if curl -fsS "https://api.github.com/repos/nextstrain/${repo}/dispatches" \
+if curl -fsS "https://api.github.com/repos/${github_repo}/dispatches" \
     -H 'Accept: application/vnd.github.v3+json' \
     -H 'Content-Type: application/json' \
     -H "$auth" \

ingest/vendored/trigger-on-new-data

@@ -5,7 +5,7 @@ set -euo pipefail
 
 bin="$(dirname "$0")"
 
-repo="${1:?A repository name is required as the first argument.}"
+github_repo="${1:?A GitHub repository with owner and repository name is required as the first argument.}"
 event_type="${2:?An event type is required as the second argument.}"
 metadata="${3:?A metadata upload output file is required as the third argument.}"
 sequences="${4:?An sequence FASTA upload output file is required as the fourth argument.}"
@@ -19,7 +19,7 @@ slack_message=""
 # grep exit status 0 for found match, 1 for no match, 2 if an error occurred
 if [[ $new_metadata -eq 1 || $new_sequences -eq 1 ]]; then
     slack_message="Triggering new builds due to updated metadata and/or sequences"
-    "$bin"/trigger "$repo" "$event_type"
+    "$bin"/trigger "$github_repo" "$event_type"
 elif [[ $new_metadata -eq 0 && $new_sequences -eq 0 ]]; then
     slack_message="Skipping trigger of rebuild: Both metadata TSV and sequences FASTA are identical to S3 files."
 else

ingest/workflow/snakemake_rules/slack_notifications.smk

@@ -30,7 +30,7 @@ rule notify_on_genbank_record_change:
         s3_src=S3_SRC,
     shell:
         """
-        ./bin/notify-on-record-change {input.genbank_ndjson} {params.s3_src:q}/genbank.ndjson.xz Genbank
+        ./vendored/notify-on-record-change {input.genbank_ndjson} {params.s3_src:q}/genbank.ndjson.xz Genbank
         """
 
 
@@ -43,7 +43,7 @@ rule notify_on_metadata_diff:
         s3_src=S3_SRC,
     shell:
         """
-        ./bin/notify-on-diff {input.metadata} {params.s3_src:q}/metadata.tsv.gz
+        ./vendored/notify-on-diff {input.metadata} {params.s3_src:q}/metadata.tsv.gz
         """
 
 

ingest/workflow/snakemake_rules/trigger_rebuild.smk

@@ -15,5 +15,5 @@ rule trigger_build:
         touch("data/trigger/rebuild.done")
     shell:
         """
-        ./bin/trigger-on-new-data {input.metadata_upload} {input.fasta_upload}
+        ./vendored/trigger-on-new-data nextstrain/monkeypox rebuild {input.metadata_upload} {input.fasta_upload}
         """

ingest/workflow/snakemake_rules/upload.smk

@@ -56,7 +56,7 @@ rule upload_to_s3:
         cloudfront_domain=config["upload"].get("s3", {}).get("cloudfront_domain", ""),
     shell:
         """
-        ./bin/upload-to-s3 \
+        ./vendored/upload-to-s3 \
             {params.quiet} \
             {input.file_to_upload:q} \
             {params.s3_dst:q}/{wildcards.remote_file_name:q} \