Rsync transfers from web to download need to be prioritized

lehtiolab · Feb 10, 2025 · 019e241 · 019e241
1 parent 37067fb
commit 019e241
Show file tree

Hide file tree

Showing 4 changed files with 5 additions and 2 deletions.
diff --git a/deploy/default_inventory/group_vars/all b/deploy/default_inventory/group_vars/all
@@ -22,6 +22,7 @@ celeryenv_file: "{{ kanteledir }}/celery.env"
 kantelelogdir: "{{ kanteledir }}/log"
 
 queue_local: mv_md5_storage
+queue_web_rsync: web_rsync
 queue_dl: file_download
 queue_backup: backup_archive
 queue_scan: scaninbox
diff --git a/deploy/templates/celery_storage_env.j2 b/deploy/templates/celery_storage_env.j2
@@ -2,7 +2,7 @@ CELERYD_NODES="local download backup scaninbox"
 CELERY_BIN="{{ kanteledir }}/venv/bin/celery"
 CELERYD_MULTI="multi"
 CELERY_APP="kantele"
-CELERYD_OPTS="-c:local 4 -c:download 1 -c:backup 2 -c:scaninbox 1 -Q:local {{ queue_local }} -Q:download {{ queue_dl }} -Q:backup {{ queue_backup }} -Q:scaninbox {{ queue_scan }} --statedb={{ kantelelogdir }}/%n%I.state"
+CELERYD_OPTS="-c:local 4 -c:download 1 -c:webrsync 1 -c:backup 2 -c:scaninbox 1 -Q:local {{ queue_local }} -Q:download {{ queue_dl }} -Q:webrsync {{ queue_web_rsync }} -Q:backup {{ queue_backup }} -Q:scaninbox {{ queue_scan }} --statedb={{ kantelelogdir }}/%n%I.state"
 CELERYD_LOG_LEVEL="INFO"
 CELERYD_LOG_FILE="{{ kantelelogdir }}/%n%I.log"
 CELERYD_PID_FILE="{{ kantelelogdir }}/%n%I.pid"
@@ -34,6 +34,7 @@ SECONDARY_STORAGE_RSYNC_USER='{{ secondary_storage_user }}'
 SECONDARY_STORAGE_RSYNC_KEY='{{ secondary_storage_sshkey }}'
 
 QUEUE_STORAGE="{{ queue_local }}"
+QUEUE_WEB_RSYNC ="{{ queue_web_rsync }}"
 QUEUE_FILE_DOWNLOAD="{{ queue_dl }}"
 QUEUE_BACKUP="{{ queue_backup }}"
 QUEUE_SEARCH_INBOX="{{ queue_scan }}"
diff --git a/src/backend/kantele/settings.py b/src/backend/kantele/settings.py
@@ -32,6 +32,7 @@
 CLIENT_APIKEYS = [STORAGECLIENT_APIKEY, ANALYSISCLIENT_APIKEY, ADMIN_APIKEY]
 QUEUE_STORAGE = os.environ.get('QUEUE_STORAGE', 'mv_md5_storage')
 QUEUE_FILE_DOWNLOAD = os.environ.get('QUEUE_FILE_DOWNLOAD', 'file_download')
+QUEUE_WEB_RSYNC = os.environ.get('QUEUE_FILE_DOWNLOAD', 'web_rsync')
 QUEUE_BACKUP = os.environ.get('QUEUE_BACKUP', 'backup_archive')
 QUEUE_NXF = os.environ.get('QUEUE_NXF', 'nextflow')
 QUEUE_QC_NXF = os.environ.get('QUEUE_QC_NXF', 'qc_nextflow')

diff --git a/src/backend/rawstatus/tasks.py b/src/backend/rawstatus/tasks.py
@@ -121,7 +121,7 @@ def download_px_file_raw(self, ftpurl, ftpnetloc, sf_id, raw_id, shasum, size, s
     print('MD5 of {} is {}, registered in DB'.format(dstfile, postdata['md5']))
 
 
-@shared_task(queue=settings.QUEUE_FILE_DOWNLOAD, bind=True)
+@shared_task(queue=settings.QUEUE_WEB_RSYNC, bind=True)
 def rsync_transfer_file(self, sfid, srcpath, dstpath, dstsharename, do_unzip, stablefiles):
     '''Uses rsync to transfer uploaded file from KANTELEHOST/other RSYNC_HOST to storage server.
     In case of a zipped folder transfer, the file is unzipped and an MD5 check is done