New nf-core role to pull methylseq, rnaseq, atacseq etc and other changes

host_vars/127.0.0.1/main.yml

@@ -44,9 +44,7 @@ ngi_pipeline_db_sthlm: "{{ ngi_pipeline_sthlm_path }}/db/records_db_sthlm.sql"
 piper_module_version: 1.5.1
 gatk_bundle_b37: "/sw/data/uppnex/piper_references/2016-04-07/gatk_bundle/2.8/b37/"
 
-sarek_repo: "https://github.com/SciLifeLab/sarek.git"
-sarek_version: "2.3.FIX1"
-sarek_tag: "2.3"
+sarek_tag: "2.5.2"
 sarek_dest: "{{ sw_path }}/sarek"
 sarek_containers: "{{ ngi_containers }}/sarek"
 sarek_irma_config: "{{ ngi_pipeline_conf }}/sarek_irma.config"

install.yml

@@ -38,15 +38,12 @@
     - { role: arteria-siswrap-ws, tags: arteria-siswrap }
     - { role: arteria-delivery-ws, tags: arteria-delivery }
     - { role: sarek, tags: sarek }
-    - { role: rnaseq, tags: rnaseq }
-    - { role: methylseq, tags: methylseq }
     - { role: neutronstar, tags: neutronstar }
     - { role: standalone_scripts, tags: standalone_scripts }
     - { role: ugc, tags: ugc }
     - { role: misc-tools, tags: misc-tools }
     - { role: archive-upload-ws, tags: archive-upload }
-    - { role: ampliseq, tags: ampliseq }
-    - { role: atacseq, tags: atacseq }
+    - { role: nf-core, tags: nf-core }
 
   environment: "{{ anaconda_env }}"
 

roles/multiqc/defaults/main.yml

@@ -1,6 +1,6 @@
 multiqc_repo: https://github.com/ewels/MultiQC.git
 multiqc_dest: "{{ sw_path }}/multiqc"
-multiqc_version: "v1.7"
+multiqc_version: "v1.8"
 
 multiqc_ngi_repo: https://github.com/NationalGenomicsInfrastructure/MultiQC_NGI.git
 multiqc_ngi_dest: "{{ sw_path }}/multiqc_ngi"

roles/neutronstar/defaults/main.yml

@@ -4,8 +4,8 @@ neutronstar_dest: "{{ sw_path }}/neutronstar"
 # neutronstar_container_url: "docker://nf-core/neutronstar:{{ rnaseq_version }}"
 neutronstar_container_path: "{{ ngi_containers }}/neutronstar"
 # neutronstar_image: "neutronstar-{{ neutronstar_version }}.simg"
-neutronstar_version: "157a0c904fd5c8de6a174a184aee4a0ac96a871b"
+neutronstar_version: "1.0.0"
 
 # Temporary variables whilst under dev by Remi
-neutronstar_container_url: "docker://remiolsen/neutronstar"
+neutronstar_container_url: "docker://nfcore/neutronstar"
 neutronstar_image: "neutronstar.simg"

roles/neutronstar/tasks/main.yml

@@ -46,7 +46,7 @@
   - { site: "sthlm", script: "{{ bash_env_sthlm_script }}" }
   - { site: "upps", script: "{{ bash_env_upps_script }}" }
 
-- name: Store neutronstar tool version in deployement
+- name: Store neutronstar tool version in deployment
   lineinfile:
     dest: "{{ deployed_tool_versions }}"
     line: "neutronstar: {{ neutronstar_version }}"

roles/nf-core/defaults/main.yml

@@ -0,0 +1,21 @@
+nextflow_java: "/sw/comp/java/x86_64/sun_jdk1.8.0_151"
+nextflow_version_tag: "v19.04.1"
+nextflow_download_url: "https://github.com/nextflow-io/nextflow/releases/download/{{ nextflow_version_tag }}/nextflow"
+nf_core_env: "/lupus/ngi/irma3/nf-core-env"
+pipelines:
+  - name: rnaseq
+    release: 1.4.2
+    containers:
+      - nf-core-rnaseq-1.4.2.simg
+  - name: methylseq
+    release: 1.4
+    containers:
+      - nf-core-methylseq-1.4.simg
+  - name: ampliseq
+    release: 1.1.2
+    containers:
+      - nf-core-ampliseq-1.1.2.simg
+  - name: atacseq
+    release: 1.1.0
+    containers:
+      - nf-core-atacseq-1.1.0.simg

roles/nf-core/tasks/main.yml

@@ -0,0 +1,72 @@
+---
+
+- name: install nf-core
+  local_action:
+    module: pip
+    virtualenv_command: /lupus/ngi/irma3/virtualenv-15.0.0/virtualenv.py
+    virtualenv: "{{ nf_core_env }}"
+    name: nf-core
+    state: present
+
+- name: Download NextFlow
+  local_action:
+    get_url url="{{ nextflow_download_url }}" dest="{{ nf_core_env }}/bin" mode="u+rwx,g=rwx"
+
+- name: Install NextFlow
+  local_action:
+    module: command
+    _raw_params: "{{ nf_core_env }}/bin/nextflow"
+  environment:
+    NXF_LAUNCHER: "/scratch"
+    NXF_HOME: "{{ nf_core_env }}/nextflow"
+    NXF_WORK: "/scratch"
+
+- name: nf-core
+  command: "{{ nf_core_env }}/bin/nf-core -v download {{ item.name }} --singularity --compress none --outdir {{ sw_path }}{{ item.name }} --release {{ item.release }}"
+  environment:
+    PATH: "{{ ansible_env.PATH }}:{{ nf_core_env }}/bin"
+  args:
+    chdir: "{{ sw_path }}"
+  with_items: "{{ pipelines }}"
+
+- name: Create directories for singularity images
+  file:
+    path: "{{ ngi_containers }}/{{ item.name }}"
+    state: directory
+    mode: g+s
+  with_items: "{{ pipelines }}"
+
+- name: Move singularity images
+  shell: "mv {{ sw_path }}{{ item.0.name }}/singularity-images/{{ item.1 }} {{ ngi_containers }}/{{ item.0.name }}/"
+  with_subelements:
+    - "{{ pipelines }}"
+    - containers
+
+- name: Create nf-core config
+  template:
+    src: "site.config.j2"
+    dest: "{{ ngi_pipeline_conf }}/{{ item.0.name }}_{{ item.1.site }}.config"
+  with_nested:
+  - "{{ pipelines }}"
+  - [ { site: "sthlm", project_id: "{{ ngi_pipeline_sthlm_delivery }}" }, { site: "upps", project_id: "{{ ngi_pipeline_upps_delivery }}" }]
+
+- name: Set alias for pipeline
+  lineinfile:
+    dest: "{{ ngi_pipeline_conf }}/{{ item.1.script }}"
+    line: >
+          alias {{ item.0.name }}='nextflow run {{ sw_path }}{{ item.0.name }}/workflow/main.nf \
+          --custom_config_base {{ sw_path }}{{ item.0.name }}/configs \
+          -profile uppmax \
+          -c {{ ngi_pipeline_conf }}/nextflow_irma_{{ item.1.site }}.config \
+          -c {{ ngi_pipeline_conf }}/{{ item.0.name }}_{{ item.1.site }}.config'
+    backup: no
+  with_nested:
+  - "{{ pipelines }}"
+  - [ { site: "sthlm", script: "{{ bash_env_sthlm_script }}" }, { site: "upps", script: "{{ bash_env_upps_script }}" } ]
+
+- name: Store tool version in deployment
+  lineinfile:
+    dest: "{{ deployed_tool_versions }}"
+    line: "{{ item.name }}: {{ item.release }}"
+  with_items:
+  - "{{ pipelines }}"

roles/nf-core/templates/site.config.j2

@@ -0,0 +1,20 @@
+process {
+  {% for i in item.0.containers %}
+  container = '{{ ngi_containers }}/{{ item.0.name }}/{{ i }}'
+  {% endfor %}
+  {% if item.0.name=='methylseq' %}
+  memory = { 15.GB * task.attempt }
+  time = { 6.h * task.attempt }
+
+  withName:qualimap {
+        cpus = { 4 * task.attempt }
+        memory = { 62.GB * task.attempt }
+        time = { 2.d * task.attempt }
+  }
+  {% endif %}
+}
+params {
+    {% if item.0.name=='rnaseq' %}
+    reverse_stranded = true
+    {% endif %} 
+}

roles/ngi_pipeline/templates/irma_ngi_config.yaml.j2

@@ -74,7 +74,7 @@ analysis:
             analysis_engine: ngi_pipeline.engines.qc_ngi
         RNA-seq:
             analysis_engine: ngi_pipeline.engines.rna_ngi
-            ngi_nf_path: {{ rnaseq_dest }}/main.nf
+            ngi_nf_path: {{ sw_path }}rnaseq/workflow/main.nf
             sthlm_ngi_conf: {{ sthlm_rna_seq_conf }}
             upps_ngi_conf: {{ upps_rna_seq_conf }}
         wgs_germline:

roles/ngi_reports/defaults/main.yml

@@ -1,7 +1,7 @@
 ---
 ngi_reports_repo: https://github.com/NationalGenomicsInfrastructure/ngi_reports.git
 ngi_reports_dest: "{{ sw_path }}/ngi_reports"
-ngi_reports_version: 4cd7e236586cc5853f0a58f28edad0a6d5e224a8
+ngi_reports_version: 42dddcd8728afc4740643de839aefaa8fcb4fb44
 ngi_reports_log: "/log/ngi_reports.log"
 ngi_reports_log_sthlm: "{{ ngi_pipeline_sthlm_path }}/{{ ngi_reports_log }}"
 ngi_reports_log_upps: "{{ ngi_pipeline_upps_path }}/{{ ngi_reports_log }}"