diff --git a/.github/workflows/add-tools-api-docs.yml b/.github/workflows/add-tools-api-docs.yml new file mode 100644 index 0000000000..546f923324 --- /dev/null +++ b/.github/workflows/add-tools-api-docs.yml @@ -0,0 +1,80 @@ +name: Open PR with tools API docs based on webhook call +on: + workflow_dispatch: + inputs: + ref_name: + description: 'The name of the release branch' + required: true + default: 'dev' + +jobs: + api-docs: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Checkout tools code + uses: actions/checkout@v4 + with: + repository: 'nf-core/tools' + ref: ${{ inputs.ref_name }} + path: 'tools' + + - name: setup node + uses: actions/setup-node@v4 + with: + node-version: '20' + + - name: Install dependencies + run: | + npm install remark unist-util-visit + + - name: run API generation script + run: | + cd tools + bash docs/api/generate-api-docs.sh --release ${{ inputs.ref_name }} + + # Install and run pre-commit + - uses: actions/setup-python@0a5c61591373683505ea898e09a3ea4f39ef2b9c # v5 + with: + python-version: 3.11 + + # install prettier plugins + - uses: actions/setup-node@b39b52d1213e96004bfcb1c61a8a6fa8ab84f3e8 # v4 + with: + node-version: 20 + + - run: npm install --only=dev + + - name: Install pre-commit + run: pip install pre-commit + + - name: Run pre-commit + id: pre-commit + run: pre-commit run --all-files + continue-on-error: true + + - name: Commit & push changes + id: commit-and-push + run: | + git config user.email "core@nf-co.re" + git config user.name "nf-core-bot" + git config push.default upstream + git add . + git status + git commit -m "[automated] Update nf-core/tools API docs for ${{ inputs.ref_name }}" + git push + + - name: Create PR + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + commit-message: Update nf-core/tools API docs for ${{github.ref_name}} + title: Update nf-core/tools API docs for ${{github.ref_name}} + body: | + This PR updates the nf-core/tools API docs for ${{github.ref_name}}. + Please review the changes and merge if they look good. + branch: 'update-api-docs-${{github.ref_name}}' + delete-branch: true + base: 'main' + draft: false diff --git a/astro.config.mjs b/astro.config.mjs index 858aab3785..b1c4065743 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -30,34 +30,40 @@ import icon from "astro-icon"; const latestToolsRelease = await fetch('https://api.github.com/repos/nf-core/tools/releases/latest') .then((res) => res.json()) .then((json) => json.tag_name); -let latestPipelineReleases = {}; +let latestPipelineReleases = {}; pipelines_json.remote_workflows.map( (pipeline) => (latestPipelineReleases[pipeline.name] = `/${pipeline.name}/${pipeline.releases[0].tag_name}/`), ); -const latestTollsURL = `/tools/docs/'+${latestToolsRelease}`; // https://astro.build/config export default defineConfig({ site: 'https://nf-co.re/', output: 'hybrid', adapter: netlify(), redirects: { - [latestTollsURL]: 'https://oldsite.nf-co.re/tools/docs/latest/', ...latestPipelineReleases, }, integrations: [ svelte(), icon({ - include:{ - // only include a subset of icons - "file-icons":["nextflow"], - logos:["twitter","mastodon-icon","slack-icon","aws","microsoft-azure","github-actions", "youtube-icon"], - fa:["github"], - "fa-brands":["github"], - "line-md":["check-list-3-twotone"], - "mdi":["aws","slack","youtube"], - octicon:["link-external-16","table-16"], - } + include: { + // only include a subset of icons + 'file-icons': ['nextflow'], + logos: [ + 'twitter', + 'mastodon-icon', + 'slack-icon', + 'aws', + 'microsoft-azure', + 'github-actions', + 'youtube-icon', + ], + fa: ['github'], + 'fa-brands': ['github'], + 'line-md': ['check-list-3-twotone'], + mdi: ['aws', 'slack', 'youtube'], + octicon: ['link-external-16', 'table-16'], + }, }), sitemap(), prefetch(), @@ -190,7 +196,7 @@ export default defineConfig({ theme: { dark: 'github-dark', light: 'github-light', - } + }, }, ], rehypeKatex, diff --git a/public/_redirects b/public/_redirects index a4162e200e..4567883800 100644 --- a/public/_redirects +++ b/public/_redirects @@ -3,9 +3,6 @@ # Temporary oldsite redirections /logo/* https://oldsite.nf-co.re/logo/:splat 200 - -/tools-docs/lint_tests/* https://oldsite.nf-co.re/tools/docs/latest/pipeline_lint_tests/:splat -/tools/docs/* https://oldsite.nf-co.re/tools/docs/:splat /pipeline_schema_builder/* https://oldsite.nf-co.re/pipeline_schema_builder/:splat 200 /launch/* https://oldsite.nf-co.re/launch/:splat 200 /assets/* https://oldsite.nf-co.re/assets/:splat 200 @@ -16,4 +13,5 @@ /developers/* /docs/contributing/:splat /contributing/* /docs/contributing/:splat /assets/img/* /images/:splat +/tools-docs/lint_tests/* /tools/docs/latest/pipeline_lint_tests/:splat diff --git a/src/components/navbar/Navbar.astro b/src/components/navbar/Navbar.astro index b90531bfae..e4efc5812b 100644 --- a/src/components/navbar/Navbar.astro +++ b/src/components/navbar/Navbar.astro @@ -107,6 +107,9 @@ newBlogPost = await getCollection('blog').then((posts) => {
  • Configs & tools
  • Shared configs
  • Tools
    • +
  • + Linting Errors +
  • diff --git a/src/components/sidebar/SidebarNav.astro b/src/components/sidebar/SidebarNav.astro index 1606e6307c..b750a708ad 100644 --- a/src/components/sidebar/SidebarNav.astro +++ b/src/components/sidebar/SidebarNav.astro @@ -1,5 +1,5 @@ --- -const { items, minimal } = Astro.props; +const { items, minimal, accordion } = Astro.props; export interface Props { items: { @@ -12,13 +12,14 @@ export interface Props { }[]; }[]; minimal?: boolean; + accordion?: boolean; } let url = Astro.url.pathname.endsWith('/') ? Astro.url.pathname.slice(0, -1) : Astro.url.pathname; url = url.replace(/\.html$/, ''); --- -
    +
    { items && @@ -68,9 +69,8 @@ url = url.replace(/\.html$/, ''); 'stretched-link', ]} href={child.slug} - > - {child.title} - + /> +
    • ); })} @@ -185,4 +185,17 @@ url = url.replace(/\.html$/, ''); } } } + // show only the active section + .nav.accordion { + ul { + display: none; + } + h6:has(.active) ~ ul, + ul:has(.active) { + display: block; + } + div.mb-3:not(:has(.active)) { + margin-bottom: 0 !important; + } + } diff --git a/src/components/sidebar/SidebarToc.svelte b/src/components/sidebar/SidebarToc.svelte index 4af946fe8b..d9a4b9ebac 100644 --- a/src/components/sidebar/SidebarToc.svelte +++ b/src/components/sidebar/SidebarToc.svelte @@ -24,7 +24,7 @@ // make margin classes from min to max heading depth let headingMargin = {}; for (let i = minHeadingDepth; i <= 4; i++) { - headingMargin[i] = 'ps-' + (i - minHeadingDepth) * 2; + headingMargin[i] = 'ps-' + (i - minHeadingDepth); } let activeHeading = {}; onMount(() => { @@ -47,7 +47,7 @@
    -
    +
    {#if headings.length > 1} On this page {/if} @@ -57,7 +57,7 @@
      {#each headings as heading (heading)}
    • diff --git a/src/content/config.ts b/src/content/config.ts index a4de3ad68d..474761451b 100644 --- a/src/content/config.ts +++ b/src/content/config.ts @@ -124,10 +124,13 @@ const blog = defineCollection({ const pipelines = defineCollection({}); +const tools = defineCollection({}); + export const collections = { events: events, docs: docs, about: about, pipelines: pipelines, blog: blog, + tools: tools, }; diff --git a/src/content/tools/docs/1.10.1/bump_version.md b/src/content/tools/docs/1.10.1/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.10.1/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.10.1/create.md b/src/content/tools/docs/1.10.1/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.10.1/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.10.1/download.md b/src/content/tools/docs/1.10.1/download.md new file mode 100644 index 0000000000..fbea76fc98 --- /dev/null +++ b/src/content/tools/docs/1.10.1/download.md @@ -0,0 +1,68 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.10.1/index.md b/src/content/tools/docs/1.10.1/index.md new file mode 100644 index 0000000000..2ad86b1300 --- /dev/null +++ b/src/content/tools/docs/1.10.1/index.md @@ -0,0 +1,41 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.10.1/licences.md b/src/content/tools/docs/1.10.1/licences.md new file mode 100644 index 0000000000..dcc395683a --- /dev/null +++ b/src/content/tools/docs/1.10.1/licences.md @@ -0,0 +1,47 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.10.1/lint.md b/src/content/tools/docs/1.10.1/lint.md new file mode 100644 index 0000000000..754628ce81 --- /dev/null +++ b/src/content/tools/docs/1.10.1/lint.md @@ -0,0 +1,372 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +Makes sure it is triggered only on `release`. + +#### `check_actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +Makes sure it is triggered only on `push` to `master`. + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub Actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub Actions CI workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub Actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_cookiecutter_strings(){:python}` + +Look for the string ‘cookiecutter’ in all pipeline files. +Finding it probably means that there has been a copy+paste error from the template. + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'nextflow_schema.json', +'Dockerfile', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workflows/linting.yml' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +At least one string in each list must be present for fail and warn. +Any config in config_fail_ifdefined results in a failure. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_schema_lint(){:python}` + +Lint the pipeline schema + +#### `check_schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container` (if set) and `$GITHUB_REF` (if a GitHub Actions release). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `get_results_md(){:python}` + +Function to create a markdown file suitable for posting in a GitHub comment + +#### `github_comment(){:python}` + +If we are running in a GitHub PR, try to post results as a comment + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +#### `save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.10.1/list.md b/src/content/tools/docs/1.10.1/list.md new file mode 100644 index 0000000000..0e4cec26f3 --- /dev/null +++ b/src/content/tools/docs/1.10.1/list.md @@ -0,0 +1,93 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.10.1/utils.md b/src/content/tools/docs/1.10.1/utils.md new file mode 100644 index 0000000000..7988a40936 --- /dev/null +++ b/src/content/tools/docs/1.10.1/utils.md @@ -0,0 +1,46 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.10.1/workflow.md b/src/content/tools/docs/1.10.1/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.10.1/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.10.2/bump_version.md b/src/content/tools/docs/1.10.2/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.10.2/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.10.2/create.md b/src/content/tools/docs/1.10.2/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.10.2/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.10.2/download.md b/src/content/tools/docs/1.10.2/download.md new file mode 100644 index 0000000000..fbea76fc98 --- /dev/null +++ b/src/content/tools/docs/1.10.2/download.md @@ -0,0 +1,68 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.10.2/index.md b/src/content/tools/docs/1.10.2/index.md new file mode 100644 index 0000000000..2ad86b1300 --- /dev/null +++ b/src/content/tools/docs/1.10.2/index.md @@ -0,0 +1,41 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.10.2/licences.md b/src/content/tools/docs/1.10.2/licences.md new file mode 100644 index 0000000000..dcc395683a --- /dev/null +++ b/src/content/tools/docs/1.10.2/licences.md @@ -0,0 +1,47 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.10.2/lint.md b/src/content/tools/docs/1.10.2/lint.md new file mode 100644 index 0000000000..754628ce81 --- /dev/null +++ b/src/content/tools/docs/1.10.2/lint.md @@ -0,0 +1,372 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +Makes sure it is triggered only on `release`. + +#### `check_actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +Makes sure it is triggered only on `push` to `master`. + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub Actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub Actions CI workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub Actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_cookiecutter_strings(){:python}` + +Look for the string ‘cookiecutter’ in all pipeline files. +Finding it probably means that there has been a copy+paste error from the template. + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'nextflow_schema.json', +'Dockerfile', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workflows/linting.yml' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +At least one string in each list must be present for fail and warn. +Any config in config_fail_ifdefined results in a failure. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_schema_lint(){:python}` + +Lint the pipeline schema + +#### `check_schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container` (if set) and `$GITHUB_REF` (if a GitHub Actions release). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `get_results_md(){:python}` + +Function to create a markdown file suitable for posting in a GitHub comment + +#### `github_comment(){:python}` + +If we are running in a GitHub PR, try to post results as a comment + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +#### `save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.10.2/list.md b/src/content/tools/docs/1.10.2/list.md new file mode 100644 index 0000000000..0e4cec26f3 --- /dev/null +++ b/src/content/tools/docs/1.10.2/list.md @@ -0,0 +1,93 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.10.2/utils.md b/src/content/tools/docs/1.10.2/utils.md new file mode 100644 index 0000000000..7988a40936 --- /dev/null +++ b/src/content/tools/docs/1.10.2/utils.md @@ -0,0 +1,46 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.10.2/workflow.md b/src/content/tools/docs/1.10.2/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.10.2/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.10/bump_version.md b/src/content/tools/docs/1.10/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.10/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.10/create.md b/src/content/tools/docs/1.10/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.10/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.10/download.md b/src/content/tools/docs/1.10/download.md new file mode 100644 index 0000000000..fbea76fc98 --- /dev/null +++ b/src/content/tools/docs/1.10/download.md @@ -0,0 +1,68 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.10/index.md b/src/content/tools/docs/1.10/index.md new file mode 100644 index 0000000000..2ad86b1300 --- /dev/null +++ b/src/content/tools/docs/1.10/index.md @@ -0,0 +1,41 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.10/licences.md b/src/content/tools/docs/1.10/licences.md new file mode 100644 index 0000000000..dcc395683a --- /dev/null +++ b/src/content/tools/docs/1.10/licences.md @@ -0,0 +1,47 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.10/lint.md b/src/content/tools/docs/1.10/lint.md new file mode 100644 index 0000000000..10a9c10c22 --- /dev/null +++ b/src/content/tools/docs/1.10/lint.md @@ -0,0 +1,372 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +Makes sure it is triggered only on `release`. + +#### `check_actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +Makes sure it is triggered only on `push` to `master`. + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub Actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub Actions CI workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub Actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_cookiecutter_strings(){:python}` + +Look for the string ‘cookiecutter’ in all pipeline files. +Finding it probably means that there has been a copy+paste error from the template. + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'nextflow_schema.json', +'Dockerfile', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workflows/linting.yml' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +At least one string in each list must be present for fail and warn. +Any config in config_fail_ifdefined results in a failure. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_schema_lint(){:python}` + +Lint the pipeline schema + +#### `check_schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container` (if set) and `$GITHUB_REF` (if a GitHub Actions release). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `get_results_md(){:python}` + +Function to create a markdown file suitable for posting in a GitHub comment + +#### `github_comment(){:python}` + +If we are running in a GitHub PR, try to post results as a comment + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +#### `save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.10/list.md b/src/content/tools/docs/1.10/list.md new file mode 100644 index 0000000000..0e4cec26f3 --- /dev/null +++ b/src/content/tools/docs/1.10/list.md @@ -0,0 +1,93 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.10/utils.md b/src/content/tools/docs/1.10/utils.md new file mode 100644 index 0000000000..7988a40936 --- /dev/null +++ b/src/content/tools/docs/1.10/utils.md @@ -0,0 +1,46 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.10/workflow.md b/src/content/tools/docs/1.10/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.10/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.11/bump_version.md b/src/content/tools/docs/1.11/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.11/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.11/create.md b/src/content/tools/docs/1.11/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.11/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.11/download.md b/src/content/tools/docs/1.11/download.md new file mode 100644 index 0000000000..fbea76fc98 --- /dev/null +++ b/src/content/tools/docs/1.11/download.md @@ -0,0 +1,68 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.11/index.md b/src/content/tools/docs/1.11/index.md new file mode 100644 index 0000000000..2ad86b1300 --- /dev/null +++ b/src/content/tools/docs/1.11/index.md @@ -0,0 +1,41 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.11/licences.md b/src/content/tools/docs/1.11/licences.md new file mode 100644 index 0000000000..dcc395683a --- /dev/null +++ b/src/content/tools/docs/1.11/licences.md @@ -0,0 +1,47 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.11/lint.md b/src/content/tools/docs/1.11/lint.md new file mode 100644 index 0000000000..8a4c15d975 --- /dev/null +++ b/src/content/tools/docs/1.11/lint.md @@ -0,0 +1,374 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +Makes sure it is triggered only on `release` and workflow_dispatch. + +#### `check_actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +Makes sure it is triggered only on `push` to `master`. + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub Actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub Actions CI workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub Actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_cookiecutter_strings(){:python}` + +Look for the string ‘cookiecutter’ in all pipeline files. +Finding it probably means that there has been a copy+paste error from the template. + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'nextflow_schema.json', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workflows/linting.yml' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'Dockerfile', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity', +'parameters.settings.json', +'bin/markdown_to_html.r' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +At least one string in each list must be present for fail and warn. +Any config in config_fail_ifdefined results in a failure. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_schema_lint(){:python}` + +Lint the pipeline schema + +#### `check_schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container` (if set) and `$GITHUB_REF` (if a GitHub Actions release). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `get_results_md(){:python}` + +Function to create a markdown file suitable for posting in a GitHub comment + +#### `github_comment(){:python}` + +If we are running in a GitHub PR, try to post results as a comment + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +#### `save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.11/list.md b/src/content/tools/docs/1.11/list.md new file mode 100644 index 0000000000..0e4cec26f3 --- /dev/null +++ b/src/content/tools/docs/1.11/list.md @@ -0,0 +1,93 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.11/utils.md b/src/content/tools/docs/1.11/utils.md new file mode 100644 index 0000000000..7988a40936 --- /dev/null +++ b/src/content/tools/docs/1.11/utils.md @@ -0,0 +1,46 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.11/workflow.md b/src/content/tools/docs/1.11/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.11/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.12.1/bump_version.md b/src/content/tools/docs/1.12.1/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.12.1/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.12.1/create.md b/src/content/tools/docs/1.12.1/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.12.1/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.12.1/download.md b/src/content/tools/docs/1.12.1/download.md new file mode 100644 index 0000000000..9911aabbd3 --- /dev/null +++ b/src/content/tools/docs/1.12.1/download.md @@ -0,0 +1,70 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.12.1/index.md b/src/content/tools/docs/1.12.1/index.md new file mode 100644 index 0000000000..115755e1c6 --- /dev/null +++ b/src/content/tools/docs/1.12.1/index.md @@ -0,0 +1,49 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) + - [`ModulesRepo`](modules#nf_core.modules.ModulesRepo) + - [`PipelineModules`](modules#nf_core.modules.PipelineModules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) +- [nf_core.utils](utils) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.12.1/launch.md b/src/content/tools/docs/1.12.1/launch.md new file mode 100644 index 0000000000..0369d56b93 --- /dev/null +++ b/src/content/tools/docs/1.12.1/launch.md @@ -0,0 +1,89 @@ +# nf_core.launch + + + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/1.12.1/licences.md b/src/content/tools/docs/1.12.1/licences.md new file mode 100644 index 0000000000..ca4d304135 --- /dev/null +++ b/src/content/tools/docs/1.12.1/licences.md @@ -0,0 +1,49 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.12.1/lint.md b/src/content/tools/docs/1.12.1/lint.md new file mode 100644 index 0000000000..9d1e55bc56 --- /dev/null +++ b/src/content/tools/docs/1.12.1/lint.md @@ -0,0 +1,379 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Bases: `object` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +#### `_wrap_quotes(files){:python}` + +#### `check_actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +Makes sure it is triggered only on `release` and workflow_dispatch. + +#### `check_actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +Makes sure it is triggered only on `push` to `master`. + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub Actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub Actions CI workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub Actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_cookiecutter_strings(){:python}` + +Look for the string ‘cookiecutter’ in all pipeline files. +Finding it probably means that there has been a copy+paste error from the template. + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'nextflow_schema.json', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workflows/linting.yml' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'Dockerfile', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity', +'parameters.settings.json', +'bin/markdown_to_html.r', +'.github/workflows/push_dockerhub.yml' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +At least one string in each list must be present for fail and warn. +Any config in config_fail_ifdefined results in a failure. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_schema_lint(){:python}` + +Lint the pipeline schema + +#### `check_schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container` (if set) and `$GITHUB_REF` (if a GitHub Actions release). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `get_results_md(){:python}` + +Function to create a markdown file suitable for posting in a GitHub comment + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +#### `print_results(show_passed=False){:python}` + +#### `save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.12.1/list.md b/src/content/tools/docs/1.12.1/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/1.12.1/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.12.1/modules.md b/src/content/tools/docs/1.12.1/modules.md new file mode 100644 index 0000000000..74fa1142ac --- /dev/null +++ b/src/content/tools/docs/1.12.1/modules.md @@ -0,0 +1,73 @@ +# nf_core.modules + + + +Code to handle DSL2 module imports from a GitHub repository + +### _`class{:python}`_`nf_core.modules.ModulesRepo(repo='nf-core/modules', branch='master'){:python}` + +Bases: `object` + +An object to store details about the repository being used for modules. + +Used by the nf-core modules top-level command with -r and -b flags, +so that this can be used in the same way by all sucommands. + +### _`class{:python}`_`nf_core.modules.PipelineModules{:python}` + +Bases: `object` + +#### `check_modules(){:python}` + +#### `download_gh_file(dl_filename, api_url){:python}` + +Download a file from GitHub using the GitHub API + +- **Parameters:** + - **dl_filename** (_string_) – Path to save file to + - **api_url** (_string_) – GitHub API URL for file +- **Raises:** + **If a problem**\*\*,\*\* **raises an error** – + +#### `get_module_file_urls(module){:python}` + +Fetch list of URLs for a specific module + +Takes the name of a module and iterates over the GitHub repo file tree. +Loops over items that are prefixed with the path ‘software/\’ and ignores +anything that’s not a blob. Also ignores the test/ subfolder. + +Returns a dictionary with keys as filenames and values as GitHub API URIs. +These can be used to then download file contents. + +- **Parameters:** + **module** (_string_) – Name of module for which to fetch a set of URLs +- **Returns:** + Set of files and associated URLs as follows: + { + : ‘software/fastqc/main.nf’: ‘’, + ‘software/fastqc/meta.yml’: ‘’ + + } + +- **Return type:** + dict + +#### `get_modules_file_tree(){:python}` + +Fetch the file list from the repo, using the GitHub API + +Sets self.modules_file_tree +: self.modules_current_hash +self.modules_avail_module_names + +#### `install(module){:python}` + +#### `list_modules(){:python}` + +Get available module names from GitHub tree for repo +and print as list to stdout + +#### `remove(module){:python}` + +#### `update(module, force=False){:python}` diff --git a/src/content/tools/docs/1.12.1/schema.md b/src/content/tools/docs/1.12.1/schema.md new file mode 100644 index 0000000000..6e74442246 --- /dev/null +++ b/src/content/tools/docs/1.12.1/schema.md @@ -0,0 +1,106 @@ +# nf_core.schema + + + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/1.12.1/sync.md b/src/content/tools/docs/1.12.1/sync.md new file mode 100644 index 0000000000..c12523d92b --- /dev/null +++ b/src/content/tools/docs/1.12.1/sync.md @@ -0,0 +1 @@ +# nf_core.sync diff --git a/src/content/tools/docs/1.12.1/utils.md b/src/content/tools/docs/1.12.1/utils.md new file mode 100644 index 0000000000..cc83aab208 --- /dev/null +++ b/src/content/tools/docs/1.12.1/utils.md @@ -0,0 +1,50 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.12/bump_version.md b/src/content/tools/docs/1.12/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.12/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.12/create.md b/src/content/tools/docs/1.12/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.12/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.12/download.md b/src/content/tools/docs/1.12/download.md new file mode 100644 index 0000000000..9911aabbd3 --- /dev/null +++ b/src/content/tools/docs/1.12/download.md @@ -0,0 +1,70 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.12/index.md b/src/content/tools/docs/1.12/index.md new file mode 100644 index 0000000000..c73153a0b1 --- /dev/null +++ b/src/content/tools/docs/1.12/index.md @@ -0,0 +1,48 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) + - [`ModulesRepo`](modules#nf_core.modules.ModulesRepo) + - [`PipelineModules`](modules#nf_core.modules.PipelineModules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) +- [nf_core.utils](utils) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.12/launch.md b/src/content/tools/docs/1.12/launch.md new file mode 100644 index 0000000000..0874b30c29 --- /dev/null +++ b/src/content/tools/docs/1.12/launch.md @@ -0,0 +1 @@ +# nf_core.launch diff --git a/src/content/tools/docs/1.12/licences.md b/src/content/tools/docs/1.12/licences.md new file mode 100644 index 0000000000..ca4d304135 --- /dev/null +++ b/src/content/tools/docs/1.12/licences.md @@ -0,0 +1,49 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.12/lint.md b/src/content/tools/docs/1.12/lint.md new file mode 100644 index 0000000000..9d1e55bc56 --- /dev/null +++ b/src/content/tools/docs/1.12/lint.md @@ -0,0 +1,379 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Bases: `object` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +#### `_wrap_quotes(files){:python}` + +#### `check_actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +Makes sure it is triggered only on `release` and workflow_dispatch. + +#### `check_actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +Makes sure it is triggered only on `push` to `master`. + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub Actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub Actions CI workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub Actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_cookiecutter_strings(){:python}` + +Look for the string ‘cookiecutter’ in all pipeline files. +Finding it probably means that there has been a copy+paste error from the template. + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'nextflow_schema.json', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workflows/linting.yml' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'Dockerfile', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity', +'parameters.settings.json', +'bin/markdown_to_html.r', +'.github/workflows/push_dockerhub.yml' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +At least one string in each list must be present for fail and warn. +Any config in config_fail_ifdefined results in a failure. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_schema_lint(){:python}` + +Lint the pipeline schema + +#### `check_schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container` (if set) and `$GITHUB_REF` (if a GitHub Actions release). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `get_results_md(){:python}` + +Function to create a markdown file suitable for posting in a GitHub comment + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +#### `print_results(show_passed=False){:python}` + +#### `save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, show_passed=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.12/list.md b/src/content/tools/docs/1.12/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/1.12/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.12/modules.md b/src/content/tools/docs/1.12/modules.md new file mode 100644 index 0000000000..74fa1142ac --- /dev/null +++ b/src/content/tools/docs/1.12/modules.md @@ -0,0 +1,73 @@ +# nf_core.modules + + + +Code to handle DSL2 module imports from a GitHub repository + +### _`class{:python}`_`nf_core.modules.ModulesRepo(repo='nf-core/modules', branch='master'){:python}` + +Bases: `object` + +An object to store details about the repository being used for modules. + +Used by the nf-core modules top-level command with -r and -b flags, +so that this can be used in the same way by all sucommands. + +### _`class{:python}`_`nf_core.modules.PipelineModules{:python}` + +Bases: `object` + +#### `check_modules(){:python}` + +#### `download_gh_file(dl_filename, api_url){:python}` + +Download a file from GitHub using the GitHub API + +- **Parameters:** + - **dl_filename** (_string_) – Path to save file to + - **api_url** (_string_) – GitHub API URL for file +- **Raises:** + **If a problem**\*\*,\*\* **raises an error** – + +#### `get_module_file_urls(module){:python}` + +Fetch list of URLs for a specific module + +Takes the name of a module and iterates over the GitHub repo file tree. +Loops over items that are prefixed with the path ‘software/\’ and ignores +anything that’s not a blob. Also ignores the test/ subfolder. + +Returns a dictionary with keys as filenames and values as GitHub API URIs. +These can be used to then download file contents. + +- **Parameters:** + **module** (_string_) – Name of module for which to fetch a set of URLs +- **Returns:** + Set of files and associated URLs as follows: + { + : ‘software/fastqc/main.nf’: ‘’, + ‘software/fastqc/meta.yml’: ‘’ + + } + +- **Return type:** + dict + +#### `get_modules_file_tree(){:python}` + +Fetch the file list from the repo, using the GitHub API + +Sets self.modules_file_tree +: self.modules_current_hash +self.modules_avail_module_names + +#### `install(module){:python}` + +#### `list_modules(){:python}` + +Get available module names from GitHub tree for repo +and print as list to stdout + +#### `remove(module){:python}` + +#### `update(module, force=False){:python}` diff --git a/src/content/tools/docs/1.12/schema.md b/src/content/tools/docs/1.12/schema.md new file mode 100644 index 0000000000..6e74442246 --- /dev/null +++ b/src/content/tools/docs/1.12/schema.md @@ -0,0 +1,106 @@ +# nf_core.schema + + + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/1.12/sync.md b/src/content/tools/docs/1.12/sync.md new file mode 100644 index 0000000000..c12523d92b --- /dev/null +++ b/src/content/tools/docs/1.12/sync.md @@ -0,0 +1 @@ +# nf_core.sync diff --git a/src/content/tools/docs/1.12/utils.md b/src/content/tools/docs/1.12/utils.md new file mode 100644 index 0000000000..cc83aab208 --- /dev/null +++ b/src/content/tools/docs/1.12/utils.md @@ -0,0 +1,50 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.13.1/api/bump_version.md b/src/content/tools/docs/1.13.1/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.13.1/api/create.md b/src/content/tools/docs/1.13.1/api/create.md new file mode 100644 index 0000000000..92dcb9fb5e --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practise template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/1.13.1/api/download.md b/src/content/tools/docs/1.13.1/api/download.md new file mode 100644 index 0000000000..92c33262e9 --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/download.md @@ -0,0 +1,123 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, outdir=None, compress_type='tar.gz', force=False, singularity=False, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.13.1/api/index.md b/src/content/tools/docs/1.13.1/api/index.md new file mode 100644 index 0000000000..8fcbeab99d --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/index.md @@ -0,0 +1,49 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/1.13.1/api/launch.md b/src/content/tools/docs/1.13.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/1.13.1/api/licences.md b/src/content/tools/docs/1.13.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.13.1/api/lint.md b/src/content/tools/docs/1.13.1/api/lint.md new file mode 100644 index 0000000000..97f22f4911 --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed=False){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/1.13.1/api/list.md b/src/content/tools/docs/1.13.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.13.1/api/modules.md b/src/content/tools/docs/1.13.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/1.13.1/api/schema.md b/src/content/tools/docs/1.13.1/api/schema.md new file mode 100644 index 0000000000..162788eb1a --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/schema.md @@ -0,0 +1,109 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/1.13.1/api/sync.md b/src/content/tools/docs/1.13.1/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/1.13.1/api/utils.md b/src/content/tools/docs/1.13.1/api/utils.md new file mode 100644 index 0000000000..782b715f4a --- /dev/null +++ b/src/content/tools/docs/1.13.1/api/utils.md @@ -0,0 +1,200 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, look for a container +at quay.io and returns the tag of the most recent image +that matches the package version +Sends a HTTP GET request to the quay.io API. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.13.1/index.md b/src/content/tools/docs/1.13.1/index.md new file mode 100644 index 0000000000..c5988c4254 --- /dev/null +++ b/src/content/tools/docs/1.13.1/index.md @@ -0,0 +1,18 @@ + + +# Welcome to nf-core tools API documentation! + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.13.1/lint_tests/actions_awsfulltest.md b/src/content/tools/docs/1.13.1/lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..fd32fa4a32 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`workflow_run` (after the docker hub release workflow) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `workflow_run` with `workflows: ["nf-core Docker push (release)"]` and `types: [completed]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/1.13.1/lint_tests/actions_awstest.md b/src/content/tools/docs/1.13.1/lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/1.13.1/lint_tests/actions_ci.md b/src/content/tools/docs/1.13.1/lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/1.13.1/lint_tests/actions_schema_validation.md b/src/content/tools/docs/1.13.1/lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..caa60db6c4 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/actions_schema_validation.md @@ -0,0 +1,18 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml``scripts in ``.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the JSON schema for GitHub workflows <> + +To pass this test, make sure that all your workflows contain the required properties `on` and + +``` +`` +``` + +jobs\`\`and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/1.13.1/lint_tests/conda_dockerfile.md b/src/content/tools/docs/1.13.1/lint_tests/conda_dockerfile.md new file mode 100644 index 0000000000..388e51dd9a --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/conda_dockerfile.md @@ -0,0 +1,33 @@ +# conda_dockerfile + +#### `PipelineLint.conda_dockerfile(){:python}` + +Checks the Dockerfile for use with Conda environments + +:::note +This test only runs if there is both an `environment.yml` +and `Dockerfile` present in the pipeline root directory. +::: + +If a workflow has a conda `environment.yml` file, the `Dockerfile` should use this +to create the docker image. These files are typically very short, just creating the conda +environment inside the container. + +This linting test checks for the following: + +- All of the following lines are present in the file (where `PIPELINE` is your pipeline name): + > ```Dockerfile + > FROM nfcore/base:VERSION + > COPY environment.yml / + > RUN conda env create --quiet -f /environment.yml && conda clean -a + > RUN conda env export --name PIPELINE > PIPELINE.yml + > ENV PATH /opt/conda/envs/PIPELINE/bin:$PATH + > ``` +- That the `FROM nfcore/base:VERSION` is tagged to the most recent release of nf-core/tools + > - The linting tool compares the tag against the currently installed version of tools. + > - This line is not checked if running a development version of nf-core/tools. + +:::note +Additional lines and different metadata can be added to the `Dockerfile` +without causing this lint test to fail. +::: diff --git a/src/content/tools/docs/1.13.1/lint_tests/conda_env_yaml.md b/src/content/tools/docs/1.13.1/lint_tests/conda_env_yaml.md new file mode 100644 index 0000000000..f2dab4fba1 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/conda_env_yaml.md @@ -0,0 +1,36 @@ +# conda_env_yaml + +#### `PipelineLint.conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +:::note +This test is ignored if there is not an `environment.yml` +file present in the pipeline root directory. +::: + +DSL1 nf-core pipelines use a single Conda environment to manage all software +dependencies for a workflow. This can be used directly with `-profile conda` +and is also used in the `Dockerfile` to build a docker image. + +This test checks the conda `environment.yml` file to ensure that it follows nf-core guidelines. +Each dependency is checked using the [Anaconda API service](https://api.anaconda.org/docs). +Dependency sublists are ignored with the exception of `- pip`: these packages are also checked +for pinned version numbers and checked using the [PyPI JSON API](https://wiki.python.org/moin/PyPIJSON). + +Specifically, this lint test makes sure that: + +- The environment `name` must match the pipeline name and version + > - The pipeline name is defined in the config variable `manifest.name` + > - Replace the slash with a hyphen as environment names shouldn’t contain that character + > - Example: For `nf-core/test` version 1.4, the conda environment name should be `nf-core-test-1.4` +- All package dependencies have a specific version number pinned + > :::warning + > Remember that Conda package versions should be pinned with one equals sign (`toolname=1.1`), + > but pip uses two (`toolname==1.2`) + > ::: +- That package versions can be found and are the latest available + > - Test will go through all conda channels listed in the file, or check PyPI if `pip` + > - Conda dependencies with pinned channels (eg. `conda-forge::openjdk`) are ok too + > - In addition to the package name, the pinned version is checked + > - If a newer version is available, a warning will be reported diff --git a/src/content/tools/docs/1.13.1/lint_tests/files_exist.md b/src/content/tools/docs/1.13.1/lint_tests/files_exist.md new file mode 100644 index 0000000000..927fbadce4 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/files_exist.md @@ -0,0 +1,77 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that **must** be present: + +```default +.gitattributes +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/markdownlint.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/markdown_to_html.py +CHANGELOG.md +CODE_OF_CONDUCT.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'Dockerfile', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity', +'parameters.settings.json', +'bin/markdown_to_html.r', +'conf/aws.config', +'.github/workflows/push_dockerhub.yml' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` diff --git a/src/content/tools/docs/1.13.1/lint_tests/files_unchanged.md b/src/content/tools/docs/1.13.1/lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ea06ada29f --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +'.gitattributes', +'.github/.dockstore.yml', +'.github/CONTRIBUTING.md', +'.github/ISSUE_TEMPLATE/bug_report.md', +'.github/ISSUE_TEMPLATE/config.yml', +'.github/ISSUE_TEMPLATE/feature_request.md', +'.github/markdownlint.yml', +'.github/PULL_REQUEST_TEMPLATE.md', +'.github/workflows/branch.yml', +'.github/workflows/linting_comment.yml', +'.github/workflows/linting.yml', +'assets/email_template.html', +'assets/email_template.txt', +'assets/nf-core-PIPELINE_logo.png', +'assets/sendmail_template.txt', +'bin/markdown_to_html.py', +'CODE_OF_CONDUCT.md', +'docs/images/nf-core-PIPELINE_logo.png', +'docs/README.md', +'lib/nfcore_external_java_deps.jar' +'lib/NfcoreSchema.groovy', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +'.github/workflows/push_dockerhub_dev.yml', +'.github/workflows/push_dockerhub_release.yml', +'.gitignore', +'assets/multiqc_config.yaml', +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/1.13.1/lint_tests/index.md b/src/content/tools/docs/1.13.1/lint_tests/index.md new file mode 100644 index 0000000000..f0e9f7e6a1 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/index.md @@ -0,0 +1,38 @@ +# Lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [conda_dockerfile](conda_dockerfile) + - [`PipelineLint.conda_dockerfile()`](conda_dockerfile#nf_core.lint.PipelineLint.conda_dockerfile) +- [conda_env_yaml](conda_env_yaml) + - [`PipelineLint.conda_env_yaml()`](conda_env_yaml#nf_core.lint.PipelineLint.conda_env_yaml) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/1.13.1/lint_tests/merge_markers.md b/src/content/tools/docs/1.13.1/lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/1.13.1/lint_tests/nextflow_config.md b/src/content/tools/docs/1.13.1/lint_tests/nextflow_config.md new file mode 100644 index 0000000000..7f77f40c66 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/nextflow_config.md @@ -0,0 +1,65 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) +- `process.container` + - Docker Hub handle for a single default container for use by all processes. + - Must specify a tag that matches the pipeline version number if set. + - If the pipeline version number contains the string `dev`, the DockerHub tag must be `:dev` + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. diff --git a/src/content/tools/docs/1.13.1/lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/1.13.1/lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/1.13.1/lint_tests/pipeline_todos.md b/src/content/tools/docs/1.13.1/lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/1.13.1/lint_tests/readme.md b/src/content/tools/docs/1.13.1/lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/1.13.1/lint_tests/schema_lint.md b/src/content/tools/docs/1.13.1/lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/1.13.1/lint_tests/schema_params.md b/src/content/tools/docs/1.13.1/lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/1.13.1/lint_tests/template_strings.md b/src/content/tools/docs/1.13.1/lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/1.13.1/lint_tests/version_consistency.md b/src/content/tools/docs/1.13.1/lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/1.13.1/lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/1.13.2/api/bump_version.md b/src/content/tools/docs/1.13.2/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.13.2/api/create.md b/src/content/tools/docs/1.13.2/api/create.md new file mode 100644 index 0000000000..92dcb9fb5e --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practise template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/1.13.2/api/download.md b/src/content/tools/docs/1.13.2/api/download.md new file mode 100644 index 0000000000..92c33262e9 --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/download.md @@ -0,0 +1,123 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, outdir=None, compress_type='tar.gz', force=False, singularity=False, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.13.2/api/index.md b/src/content/tools/docs/1.13.2/api/index.md new file mode 100644 index 0000000000..8fcbeab99d --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/index.md @@ -0,0 +1,49 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/1.13.2/api/launch.md b/src/content/tools/docs/1.13.2/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/1.13.2/api/licences.md b/src/content/tools/docs/1.13.2/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.13.2/api/lint.md b/src/content/tools/docs/1.13.2/api/lint.md new file mode 100644 index 0000000000..97f22f4911 --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed=False){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/1.13.2/api/list.md b/src/content/tools/docs/1.13.2/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.13.2/api/modules.md b/src/content/tools/docs/1.13.2/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/1.13.2/api/schema.md b/src/content/tools/docs/1.13.2/api/schema.md new file mode 100644 index 0000000000..162788eb1a --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/schema.md @@ -0,0 +1,109 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/1.13.2/api/sync.md b/src/content/tools/docs/1.13.2/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/1.13.2/api/utils.md b/src/content/tools/docs/1.13.2/api/utils.md new file mode 100644 index 0000000000..782b715f4a --- /dev/null +++ b/src/content/tools/docs/1.13.2/api/utils.md @@ -0,0 +1,200 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, look for a container +at quay.io and returns the tag of the most recent image +that matches the package version +Sends a HTTP GET request to the quay.io API. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.13.2/index.md b/src/content/tools/docs/1.13.2/index.md new file mode 100644 index 0000000000..c5988c4254 --- /dev/null +++ b/src/content/tools/docs/1.13.2/index.md @@ -0,0 +1,18 @@ + + +# Welcome to nf-core tools API documentation! + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.13.2/lint_tests/actions_awsfulltest.md b/src/content/tools/docs/1.13.2/lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..fd32fa4a32 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`workflow_run` (after the docker hub release workflow) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `workflow_run` with `workflows: ["nf-core Docker push (release)"]` and `types: [completed]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/1.13.2/lint_tests/actions_awstest.md b/src/content/tools/docs/1.13.2/lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/1.13.2/lint_tests/actions_ci.md b/src/content/tools/docs/1.13.2/lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/1.13.2/lint_tests/actions_schema_validation.md b/src/content/tools/docs/1.13.2/lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..caa60db6c4 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/actions_schema_validation.md @@ -0,0 +1,18 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml``scripts in ``.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the JSON schema for GitHub workflows <> + +To pass this test, make sure that all your workflows contain the required properties `on` and + +``` +`` +``` + +jobs\`\`and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/1.13.2/lint_tests/conda_dockerfile.md b/src/content/tools/docs/1.13.2/lint_tests/conda_dockerfile.md new file mode 100644 index 0000000000..388e51dd9a --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/conda_dockerfile.md @@ -0,0 +1,33 @@ +# conda_dockerfile + +#### `PipelineLint.conda_dockerfile(){:python}` + +Checks the Dockerfile for use with Conda environments + +:::note +This test only runs if there is both an `environment.yml` +and `Dockerfile` present in the pipeline root directory. +::: + +If a workflow has a conda `environment.yml` file, the `Dockerfile` should use this +to create the docker image. These files are typically very short, just creating the conda +environment inside the container. + +This linting test checks for the following: + +- All of the following lines are present in the file (where `PIPELINE` is your pipeline name): + > ```Dockerfile + > FROM nfcore/base:VERSION + > COPY environment.yml / + > RUN conda env create --quiet -f /environment.yml && conda clean -a + > RUN conda env export --name PIPELINE > PIPELINE.yml + > ENV PATH /opt/conda/envs/PIPELINE/bin:$PATH + > ``` +- That the `FROM nfcore/base:VERSION` is tagged to the most recent release of nf-core/tools + > - The linting tool compares the tag against the currently installed version of tools. + > - This line is not checked if running a development version of nf-core/tools. + +:::note +Additional lines and different metadata can be added to the `Dockerfile` +without causing this lint test to fail. +::: diff --git a/src/content/tools/docs/1.13.2/lint_tests/conda_env_yaml.md b/src/content/tools/docs/1.13.2/lint_tests/conda_env_yaml.md new file mode 100644 index 0000000000..f2dab4fba1 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/conda_env_yaml.md @@ -0,0 +1,36 @@ +# conda_env_yaml + +#### `PipelineLint.conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +:::note +This test is ignored if there is not an `environment.yml` +file present in the pipeline root directory. +::: + +DSL1 nf-core pipelines use a single Conda environment to manage all software +dependencies for a workflow. This can be used directly with `-profile conda` +and is also used in the `Dockerfile` to build a docker image. + +This test checks the conda `environment.yml` file to ensure that it follows nf-core guidelines. +Each dependency is checked using the [Anaconda API service](https://api.anaconda.org/docs). +Dependency sublists are ignored with the exception of `- pip`: these packages are also checked +for pinned version numbers and checked using the [PyPI JSON API](https://wiki.python.org/moin/PyPIJSON). + +Specifically, this lint test makes sure that: + +- The environment `name` must match the pipeline name and version + > - The pipeline name is defined in the config variable `manifest.name` + > - Replace the slash with a hyphen as environment names shouldn’t contain that character + > - Example: For `nf-core/test` version 1.4, the conda environment name should be `nf-core-test-1.4` +- All package dependencies have a specific version number pinned + > :::warning + > Remember that Conda package versions should be pinned with one equals sign (`toolname=1.1`), + > but pip uses two (`toolname==1.2`) + > ::: +- That package versions can be found and are the latest available + > - Test will go through all conda channels listed in the file, or check PyPI if `pip` + > - Conda dependencies with pinned channels (eg. `conda-forge::openjdk`) are ok too + > - In addition to the package name, the pinned version is checked + > - If a newer version is available, a warning will be reported diff --git a/src/content/tools/docs/1.13.2/lint_tests/files_exist.md b/src/content/tools/docs/1.13.2/lint_tests/files_exist.md new file mode 100644 index 0000000000..927fbadce4 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/files_exist.md @@ -0,0 +1,77 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that **must** be present: + +```default +.gitattributes +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/markdownlint.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/markdown_to_html.py +CHANGELOG.md +CODE_OF_CONDUCT.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'Dockerfile', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity', +'parameters.settings.json', +'bin/markdown_to_html.r', +'conf/aws.config', +'.github/workflows/push_dockerhub.yml' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` diff --git a/src/content/tools/docs/1.13.2/lint_tests/files_unchanged.md b/src/content/tools/docs/1.13.2/lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ea06ada29f --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +'.gitattributes', +'.github/.dockstore.yml', +'.github/CONTRIBUTING.md', +'.github/ISSUE_TEMPLATE/bug_report.md', +'.github/ISSUE_TEMPLATE/config.yml', +'.github/ISSUE_TEMPLATE/feature_request.md', +'.github/markdownlint.yml', +'.github/PULL_REQUEST_TEMPLATE.md', +'.github/workflows/branch.yml', +'.github/workflows/linting_comment.yml', +'.github/workflows/linting.yml', +'assets/email_template.html', +'assets/email_template.txt', +'assets/nf-core-PIPELINE_logo.png', +'assets/sendmail_template.txt', +'bin/markdown_to_html.py', +'CODE_OF_CONDUCT.md', +'docs/images/nf-core-PIPELINE_logo.png', +'docs/README.md', +'lib/nfcore_external_java_deps.jar' +'lib/NfcoreSchema.groovy', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +'.github/workflows/push_dockerhub_dev.yml', +'.github/workflows/push_dockerhub_release.yml', +'.gitignore', +'assets/multiqc_config.yaml', +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/1.13.2/lint_tests/index.md b/src/content/tools/docs/1.13.2/lint_tests/index.md new file mode 100644 index 0000000000..f0e9f7e6a1 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/index.md @@ -0,0 +1,38 @@ +# Lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [conda_dockerfile](conda_dockerfile) + - [`PipelineLint.conda_dockerfile()`](conda_dockerfile#nf_core.lint.PipelineLint.conda_dockerfile) +- [conda_env_yaml](conda_env_yaml) + - [`PipelineLint.conda_env_yaml()`](conda_env_yaml#nf_core.lint.PipelineLint.conda_env_yaml) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/1.13.2/lint_tests/merge_markers.md b/src/content/tools/docs/1.13.2/lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/1.13.2/lint_tests/nextflow_config.md b/src/content/tools/docs/1.13.2/lint_tests/nextflow_config.md new file mode 100644 index 0000000000..7f77f40c66 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/nextflow_config.md @@ -0,0 +1,65 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) +- `process.container` + - Docker Hub handle for a single default container for use by all processes. + - Must specify a tag that matches the pipeline version number if set. + - If the pipeline version number contains the string `dev`, the DockerHub tag must be `:dev` + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. diff --git a/src/content/tools/docs/1.13.2/lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/1.13.2/lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/1.13.2/lint_tests/pipeline_todos.md b/src/content/tools/docs/1.13.2/lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/1.13.2/lint_tests/readme.md b/src/content/tools/docs/1.13.2/lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/1.13.2/lint_tests/schema_lint.md b/src/content/tools/docs/1.13.2/lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/1.13.2/lint_tests/schema_params.md b/src/content/tools/docs/1.13.2/lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/1.13.2/lint_tests/template_strings.md b/src/content/tools/docs/1.13.2/lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/1.13.2/lint_tests/version_consistency.md b/src/content/tools/docs/1.13.2/lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/1.13.2/lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/1.13.3/api/bump_version.md b/src/content/tools/docs/1.13.3/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.13.3/api/create.md b/src/content/tools/docs/1.13.3/api/create.md new file mode 100644 index 0000000000..92dcb9fb5e --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practise template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/1.13.3/api/download.md b/src/content/tools/docs/1.13.3/api/download.md new file mode 100644 index 0000000000..92c33262e9 --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/download.md @@ -0,0 +1,123 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, outdir=None, compress_type='tar.gz', force=False, singularity=False, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.13.3/api/index.md b/src/content/tools/docs/1.13.3/api/index.md new file mode 100644 index 0000000000..8fcbeab99d --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/index.md @@ -0,0 +1,49 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/1.13.3/api/launch.md b/src/content/tools/docs/1.13.3/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/1.13.3/api/licences.md b/src/content/tools/docs/1.13.3/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.13.3/api/lint.md b/src/content/tools/docs/1.13.3/api/lint.md new file mode 100644 index 0000000000..97f22f4911 --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed=False){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/1.13.3/api/list.md b/src/content/tools/docs/1.13.3/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.13.3/api/modules.md b/src/content/tools/docs/1.13.3/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/1.13.3/api/schema.md b/src/content/tools/docs/1.13.3/api/schema.md new file mode 100644 index 0000000000..162788eb1a --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/schema.md @@ -0,0 +1,109 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/1.13.3/api/sync.md b/src/content/tools/docs/1.13.3/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/1.13.3/api/utils.md b/src/content/tools/docs/1.13.3/api/utils.md new file mode 100644 index 0000000000..782b715f4a --- /dev/null +++ b/src/content/tools/docs/1.13.3/api/utils.md @@ -0,0 +1,200 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, look for a container +at quay.io and returns the tag of the most recent image +that matches the package version +Sends a HTTP GET request to the quay.io API. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.13.3/index.md b/src/content/tools/docs/1.13.3/index.md new file mode 100644 index 0000000000..c5988c4254 --- /dev/null +++ b/src/content/tools/docs/1.13.3/index.md @@ -0,0 +1,18 @@ + + +# Welcome to nf-core tools API documentation! + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.13.3/lint_tests/actions_awsfulltest.md b/src/content/tools/docs/1.13.3/lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..fd32fa4a32 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`workflow_run` (after the docker hub release workflow) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `workflow_run` with `workflows: ["nf-core Docker push (release)"]` and `types: [completed]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/1.13.3/lint_tests/actions_awstest.md b/src/content/tools/docs/1.13.3/lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/1.13.3/lint_tests/actions_ci.md b/src/content/tools/docs/1.13.3/lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/1.13.3/lint_tests/actions_schema_validation.md b/src/content/tools/docs/1.13.3/lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..caa60db6c4 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/actions_schema_validation.md @@ -0,0 +1,18 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml``scripts in ``.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the JSON schema for GitHub workflows <> + +To pass this test, make sure that all your workflows contain the required properties `on` and + +``` +`` +``` + +jobs\`\`and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/1.13.3/lint_tests/conda_dockerfile.md b/src/content/tools/docs/1.13.3/lint_tests/conda_dockerfile.md new file mode 100644 index 0000000000..388e51dd9a --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/conda_dockerfile.md @@ -0,0 +1,33 @@ +# conda_dockerfile + +#### `PipelineLint.conda_dockerfile(){:python}` + +Checks the Dockerfile for use with Conda environments + +:::note +This test only runs if there is both an `environment.yml` +and `Dockerfile` present in the pipeline root directory. +::: + +If a workflow has a conda `environment.yml` file, the `Dockerfile` should use this +to create the docker image. These files are typically very short, just creating the conda +environment inside the container. + +This linting test checks for the following: + +- All of the following lines are present in the file (where `PIPELINE` is your pipeline name): + > ```Dockerfile + > FROM nfcore/base:VERSION + > COPY environment.yml / + > RUN conda env create --quiet -f /environment.yml && conda clean -a + > RUN conda env export --name PIPELINE > PIPELINE.yml + > ENV PATH /opt/conda/envs/PIPELINE/bin:$PATH + > ``` +- That the `FROM nfcore/base:VERSION` is tagged to the most recent release of nf-core/tools + > - The linting tool compares the tag against the currently installed version of tools. + > - This line is not checked if running a development version of nf-core/tools. + +:::note +Additional lines and different metadata can be added to the `Dockerfile` +without causing this lint test to fail. +::: diff --git a/src/content/tools/docs/1.13.3/lint_tests/conda_env_yaml.md b/src/content/tools/docs/1.13.3/lint_tests/conda_env_yaml.md new file mode 100644 index 0000000000..f2dab4fba1 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/conda_env_yaml.md @@ -0,0 +1,36 @@ +# conda_env_yaml + +#### `PipelineLint.conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +:::note +This test is ignored if there is not an `environment.yml` +file present in the pipeline root directory. +::: + +DSL1 nf-core pipelines use a single Conda environment to manage all software +dependencies for a workflow. This can be used directly with `-profile conda` +and is also used in the `Dockerfile` to build a docker image. + +This test checks the conda `environment.yml` file to ensure that it follows nf-core guidelines. +Each dependency is checked using the [Anaconda API service](https://api.anaconda.org/docs). +Dependency sublists are ignored with the exception of `- pip`: these packages are also checked +for pinned version numbers and checked using the [PyPI JSON API](https://wiki.python.org/moin/PyPIJSON). + +Specifically, this lint test makes sure that: + +- The environment `name` must match the pipeline name and version + > - The pipeline name is defined in the config variable `manifest.name` + > - Replace the slash with a hyphen as environment names shouldn’t contain that character + > - Example: For `nf-core/test` version 1.4, the conda environment name should be `nf-core-test-1.4` +- All package dependencies have a specific version number pinned + > :::warning + > Remember that Conda package versions should be pinned with one equals sign (`toolname=1.1`), + > but pip uses two (`toolname==1.2`) + > ::: +- That package versions can be found and are the latest available + > - Test will go through all conda channels listed in the file, or check PyPI if `pip` + > - Conda dependencies with pinned channels (eg. `conda-forge::openjdk`) are ok too + > - In addition to the package name, the pinned version is checked + > - If a newer version is available, a warning will be reported diff --git a/src/content/tools/docs/1.13.3/lint_tests/files_exist.md b/src/content/tools/docs/1.13.3/lint_tests/files_exist.md new file mode 100644 index 0000000000..927fbadce4 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/files_exist.md @@ -0,0 +1,77 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that **must** be present: + +```default +.gitattributes +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/markdownlint.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/markdown_to_html.py +CHANGELOG.md +CODE_OF_CONDUCT.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'Dockerfile', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity', +'parameters.settings.json', +'bin/markdown_to_html.r', +'conf/aws.config', +'.github/workflows/push_dockerhub.yml' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` diff --git a/src/content/tools/docs/1.13.3/lint_tests/files_unchanged.md b/src/content/tools/docs/1.13.3/lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ea06ada29f --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +'.gitattributes', +'.github/.dockstore.yml', +'.github/CONTRIBUTING.md', +'.github/ISSUE_TEMPLATE/bug_report.md', +'.github/ISSUE_TEMPLATE/config.yml', +'.github/ISSUE_TEMPLATE/feature_request.md', +'.github/markdownlint.yml', +'.github/PULL_REQUEST_TEMPLATE.md', +'.github/workflows/branch.yml', +'.github/workflows/linting_comment.yml', +'.github/workflows/linting.yml', +'assets/email_template.html', +'assets/email_template.txt', +'assets/nf-core-PIPELINE_logo.png', +'assets/sendmail_template.txt', +'bin/markdown_to_html.py', +'CODE_OF_CONDUCT.md', +'docs/images/nf-core-PIPELINE_logo.png', +'docs/README.md', +'lib/nfcore_external_java_deps.jar' +'lib/NfcoreSchema.groovy', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +'.github/workflows/push_dockerhub_dev.yml', +'.github/workflows/push_dockerhub_release.yml', +'.gitignore', +'assets/multiqc_config.yaml', +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/1.13.3/lint_tests/index.md b/src/content/tools/docs/1.13.3/lint_tests/index.md new file mode 100644 index 0000000000..f0e9f7e6a1 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/index.md @@ -0,0 +1,38 @@ +# Lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [conda_dockerfile](conda_dockerfile) + - [`PipelineLint.conda_dockerfile()`](conda_dockerfile#nf_core.lint.PipelineLint.conda_dockerfile) +- [conda_env_yaml](conda_env_yaml) + - [`PipelineLint.conda_env_yaml()`](conda_env_yaml#nf_core.lint.PipelineLint.conda_env_yaml) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/1.13.3/lint_tests/merge_markers.md b/src/content/tools/docs/1.13.3/lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/1.13.3/lint_tests/nextflow_config.md b/src/content/tools/docs/1.13.3/lint_tests/nextflow_config.md new file mode 100644 index 0000000000..7f77f40c66 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/nextflow_config.md @@ -0,0 +1,65 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) +- `process.container` + - Docker Hub handle for a single default container for use by all processes. + - Must specify a tag that matches the pipeline version number if set. + - If the pipeline version number contains the string `dev`, the DockerHub tag must be `:dev` + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. diff --git a/src/content/tools/docs/1.13.3/lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/1.13.3/lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/1.13.3/lint_tests/pipeline_todos.md b/src/content/tools/docs/1.13.3/lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/1.13.3/lint_tests/readme.md b/src/content/tools/docs/1.13.3/lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/1.13.3/lint_tests/schema_lint.md b/src/content/tools/docs/1.13.3/lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/1.13.3/lint_tests/schema_params.md b/src/content/tools/docs/1.13.3/lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/1.13.3/lint_tests/template_strings.md b/src/content/tools/docs/1.13.3/lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/1.13.3/lint_tests/version_consistency.md b/src/content/tools/docs/1.13.3/lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/1.13.3/lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/1.13/api/bump_version.md b/src/content/tools/docs/1.13/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/1.13/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.13/api/create.md b/src/content/tools/docs/1.13/api/create.md new file mode 100644 index 0000000000..92dcb9fb5e --- /dev/null +++ b/src/content/tools/docs/1.13/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practise template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/1.13/api/download.md b/src/content/tools/docs/1.13/api/download.md new file mode 100644 index 0000000000..92c33262e9 --- /dev/null +++ b/src/content/tools/docs/1.13/api/download.md @@ -0,0 +1,123 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, outdir=None, compress_type='tar.gz', force=False, singularity=False, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.13/api/index.md b/src/content/tools/docs/1.13/api/index.md new file mode 100644 index 0000000000..8fcbeab99d --- /dev/null +++ b/src/content/tools/docs/1.13/api/index.md @@ -0,0 +1,49 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/1.13/api/launch.md b/src/content/tools/docs/1.13/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/1.13/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/1.13/api/licences.md b/src/content/tools/docs/1.13/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/1.13/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.13/api/lint.md b/src/content/tools/docs/1.13/api/lint.md new file mode 100644 index 0000000000..97f22f4911 --- /dev/null +++ b/src/content/tools/docs/1.13/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed=False){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/1.13/api/list.md b/src/content/tools/docs/1.13/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/1.13/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.13/api/modules.md b/src/content/tools/docs/1.13/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/1.13/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/1.13/api/schema.md b/src/content/tools/docs/1.13/api/schema.md new file mode 100644 index 0000000000..162788eb1a --- /dev/null +++ b/src/content/tools/docs/1.13/api/schema.md @@ -0,0 +1,109 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/1.13/api/sync.md b/src/content/tools/docs/1.13/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/1.13/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/1.13/api/utils.md b/src/content/tools/docs/1.13/api/utils.md new file mode 100644 index 0000000000..782b715f4a --- /dev/null +++ b/src/content/tools/docs/1.13/api/utils.md @@ -0,0 +1,200 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, look for a container +at quay.io and returns the tag of the most recent image +that matches the package version +Sends a HTTP GET request to the quay.io API. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.13/index.md b/src/content/tools/docs/1.13/index.md new file mode 100644 index 0000000000..c5988c4254 --- /dev/null +++ b/src/content/tools/docs/1.13/index.md @@ -0,0 +1,18 @@ + + +# Welcome to nf-core tools API documentation! + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.13/lint_tests/actions_awsfulltest.md b/src/content/tools/docs/1.13/lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..fd32fa4a32 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`workflow_run` (after the docker hub release workflow) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `workflow_run` with `workflows: ["nf-core Docker push (release)"]` and `types: [completed]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/1.13/lint_tests/actions_awstest.md b/src/content/tools/docs/1.13/lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/1.13/lint_tests/actions_ci.md b/src/content/tools/docs/1.13/lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/1.13/lint_tests/actions_schema_validation.md b/src/content/tools/docs/1.13/lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..caa60db6c4 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/actions_schema_validation.md @@ -0,0 +1,18 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml``scripts in ``.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the JSON schema for GitHub workflows <> + +To pass this test, make sure that all your workflows contain the required properties `on` and + +``` +`` +``` + +jobs\`\`and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/1.13/lint_tests/conda_dockerfile.md b/src/content/tools/docs/1.13/lint_tests/conda_dockerfile.md new file mode 100644 index 0000000000..388e51dd9a --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/conda_dockerfile.md @@ -0,0 +1,33 @@ +# conda_dockerfile + +#### `PipelineLint.conda_dockerfile(){:python}` + +Checks the Dockerfile for use with Conda environments + +:::note +This test only runs if there is both an `environment.yml` +and `Dockerfile` present in the pipeline root directory. +::: + +If a workflow has a conda `environment.yml` file, the `Dockerfile` should use this +to create the docker image. These files are typically very short, just creating the conda +environment inside the container. + +This linting test checks for the following: + +- All of the following lines are present in the file (where `PIPELINE` is your pipeline name): + > ```Dockerfile + > FROM nfcore/base:VERSION + > COPY environment.yml / + > RUN conda env create --quiet -f /environment.yml && conda clean -a + > RUN conda env export --name PIPELINE > PIPELINE.yml + > ENV PATH /opt/conda/envs/PIPELINE/bin:$PATH + > ``` +- That the `FROM nfcore/base:VERSION` is tagged to the most recent release of nf-core/tools + > - The linting tool compares the tag against the currently installed version of tools. + > - This line is not checked if running a development version of nf-core/tools. + +:::note +Additional lines and different metadata can be added to the `Dockerfile` +without causing this lint test to fail. +::: diff --git a/src/content/tools/docs/1.13/lint_tests/conda_env_yaml.md b/src/content/tools/docs/1.13/lint_tests/conda_env_yaml.md new file mode 100644 index 0000000000..f2dab4fba1 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/conda_env_yaml.md @@ -0,0 +1,36 @@ +# conda_env_yaml + +#### `PipelineLint.conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +:::note +This test is ignored if there is not an `environment.yml` +file present in the pipeline root directory. +::: + +DSL1 nf-core pipelines use a single Conda environment to manage all software +dependencies for a workflow. This can be used directly with `-profile conda` +and is also used in the `Dockerfile` to build a docker image. + +This test checks the conda `environment.yml` file to ensure that it follows nf-core guidelines. +Each dependency is checked using the [Anaconda API service](https://api.anaconda.org/docs). +Dependency sublists are ignored with the exception of `- pip`: these packages are also checked +for pinned version numbers and checked using the [PyPI JSON API](https://wiki.python.org/moin/PyPIJSON). + +Specifically, this lint test makes sure that: + +- The environment `name` must match the pipeline name and version + > - The pipeline name is defined in the config variable `manifest.name` + > - Replace the slash with a hyphen as environment names shouldn’t contain that character + > - Example: For `nf-core/test` version 1.4, the conda environment name should be `nf-core-test-1.4` +- All package dependencies have a specific version number pinned + > :::warning + > Remember that Conda package versions should be pinned with one equals sign (`toolname=1.1`), + > but pip uses two (`toolname==1.2`) + > ::: +- That package versions can be found and are the latest available + > - Test will go through all conda channels listed in the file, or check PyPI if `pip` + > - Conda dependencies with pinned channels (eg. `conda-forge::openjdk`) are ok too + > - In addition to the package name, the pinned version is checked + > - If a newer version is available, a warning will be reported diff --git a/src/content/tools/docs/1.13/lint_tests/files_exist.md b/src/content/tools/docs/1.13/lint_tests/files_exist.md new file mode 100644 index 0000000000..927fbadce4 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/files_exist.md @@ -0,0 +1,77 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that **must** be present: + +```default +.gitattributes +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/markdownlint.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/markdown_to_html.py +CHANGELOG.md +CODE_OF_CONDUCT.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'Dockerfile', +'conf/base.config', +'.github/workflows/awstest.yml', +'.github/workflows/awsfulltest.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity', +'parameters.settings.json', +'bin/markdown_to_html.r', +'conf/aws.config', +'.github/workflows/push_dockerhub.yml' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` diff --git a/src/content/tools/docs/1.13/lint_tests/files_unchanged.md b/src/content/tools/docs/1.13/lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ea06ada29f --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +'.gitattributes', +'.github/.dockstore.yml', +'.github/CONTRIBUTING.md', +'.github/ISSUE_TEMPLATE/bug_report.md', +'.github/ISSUE_TEMPLATE/config.yml', +'.github/ISSUE_TEMPLATE/feature_request.md', +'.github/markdownlint.yml', +'.github/PULL_REQUEST_TEMPLATE.md', +'.github/workflows/branch.yml', +'.github/workflows/linting_comment.yml', +'.github/workflows/linting.yml', +'assets/email_template.html', +'assets/email_template.txt', +'assets/nf-core-PIPELINE_logo.png', +'assets/sendmail_template.txt', +'bin/markdown_to_html.py', +'CODE_OF_CONDUCT.md', +'docs/images/nf-core-PIPELINE_logo.png', +'docs/README.md', +'lib/nfcore_external_java_deps.jar' +'lib/NfcoreSchema.groovy', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +'.github/workflows/push_dockerhub_dev.yml', +'.github/workflows/push_dockerhub_release.yml', +'.gitignore', +'assets/multiqc_config.yaml', +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/1.13/lint_tests/index.md b/src/content/tools/docs/1.13/lint_tests/index.md new file mode 100644 index 0000000000..f0e9f7e6a1 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/index.md @@ -0,0 +1,38 @@ +# Lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [conda_dockerfile](conda_dockerfile) + - [`PipelineLint.conda_dockerfile()`](conda_dockerfile#nf_core.lint.PipelineLint.conda_dockerfile) +- [conda_env_yaml](conda_env_yaml) + - [`PipelineLint.conda_env_yaml()`](conda_env_yaml#nf_core.lint.PipelineLint.conda_env_yaml) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/1.13/lint_tests/merge_markers.md b/src/content/tools/docs/1.13/lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/1.13/lint_tests/nextflow_config.md b/src/content/tools/docs/1.13/lint_tests/nextflow_config.md new file mode 100644 index 0000000000..7f77f40c66 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/nextflow_config.md @@ -0,0 +1,65 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) +- `process.container` + - Docker Hub handle for a single default container for use by all processes. + - Must specify a tag that matches the pipeline version number if set. + - If the pipeline version number contains the string `dev`, the DockerHub tag must be `:dev` + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. diff --git a/src/content/tools/docs/1.13/lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/1.13/lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/1.13/lint_tests/pipeline_todos.md b/src/content/tools/docs/1.13/lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/1.13/lint_tests/readme.md b/src/content/tools/docs/1.13/lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/1.13/lint_tests/schema_lint.md b/src/content/tools/docs/1.13/lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/1.13/lint_tests/schema_params.md b/src/content/tools/docs/1.13/lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/1.13/lint_tests/template_strings.md b/src/content/tools/docs/1.13/lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/1.13/lint_tests/version_consistency.md b/src/content/tools/docs/1.13/lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/1.13/lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/1.14/api/bump_version.md b/src/content/tools/docs/1.14/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/1.14/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.14/api/create.md b/src/content/tools/docs/1.14/api/create.md new file mode 100644 index 0000000000..92dcb9fb5e --- /dev/null +++ b/src/content/tools/docs/1.14/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practise template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/1.14/api/download.md b/src/content/tools/docs/1.14/api/download.md new file mode 100644 index 0000000000..7769825e08 --- /dev/null +++ b/src/content/tools/docs/1.14/api/download.md @@ -0,0 +1,142 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, release=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_release_hash(){:python}` + +Find specified release / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_release(){:python}` + +Prompt for pipeline release / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.14/api/index.md b/src/content/tools/docs/1.14/api/index.md new file mode 100644 index 0000000000..642e52b056 --- /dev/null +++ b/src/content/tools/docs/1.14/api/index.md @@ -0,0 +1,53 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/1.14/api/launch.md b/src/content/tools/docs/1.14/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/1.14/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/1.14/api/licences.md b/src/content/tools/docs/1.14/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/1.14/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/1.14/api/lint.md b/src/content/tools/docs/1.14/api/lint.md new file mode 100644 index 0000000000..e8f44cd06c --- /dev/null +++ b/src/content/tools/docs/1.14/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed=False){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/1.14/api/list.md b/src/content/tools/docs/1.14/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/1.14/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.14/api/modules.md b/src/content/tools/docs/1.14/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/1.14/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/1.14/api/schema.md b/src/content/tools/docs/1.14/api/schema.md new file mode 100644 index 0000000000..bbb2b2e39b --- /dev/null +++ b/src/content/tools/docs/1.14/api/schema.md @@ -0,0 +1,113 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/1.14/api/sync.md b/src/content/tools/docs/1.14/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/1.14/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/1.14/api/utils.md b/src/content/tools/docs/1.14/api/utils.md new file mode 100644 index 0000000000..c3761e9e96 --- /dev/null +++ b/src/content/tools/docs/1.14/api/utils.md @@ -0,0 +1,243 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, look for a container +at quay.io and returns the tag of the most recent image +that matches the package version +Sends a HTTP GET request to the quay.io API. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/1.14/index.md b/src/content/tools/docs/1.14/index.md new file mode 100644 index 0000000000..c3d07a7628 --- /dev/null +++ b/src/content/tools/docs/1.14/index.md @@ -0,0 +1,13 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..fd32fa4a32 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`workflow_run` (after the docker hub release workflow) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `workflow_run` with `workflows: ["nf-core Docker push (release)"]` and `types: [completed]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/conda_dockerfile.md b/src/content/tools/docs/1.14/pipeline_lint_tests/conda_dockerfile.md new file mode 100644 index 0000000000..388e51dd9a --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/conda_dockerfile.md @@ -0,0 +1,33 @@ +# conda_dockerfile + +#### `PipelineLint.conda_dockerfile(){:python}` + +Checks the Dockerfile for use with Conda environments + +:::note +This test only runs if there is both an `environment.yml` +and `Dockerfile` present in the pipeline root directory. +::: + +If a workflow has a conda `environment.yml` file, the `Dockerfile` should use this +to create the docker image. These files are typically very short, just creating the conda +environment inside the container. + +This linting test checks for the following: + +- All of the following lines are present in the file (where `PIPELINE` is your pipeline name): + > ```Dockerfile + > FROM nfcore/base:VERSION + > COPY environment.yml / + > RUN conda env create --quiet -f /environment.yml && conda clean -a + > RUN conda env export --name PIPELINE > PIPELINE.yml + > ENV PATH /opt/conda/envs/PIPELINE/bin:$PATH + > ``` +- That the `FROM nfcore/base:VERSION` is tagged to the most recent release of nf-core/tools + > - The linting tool compares the tag against the currently installed version of tools. + > - This line is not checked if running a development version of nf-core/tools. + +:::note +Additional lines and different metadata can be added to the `Dockerfile` +without causing this lint test to fail. +::: diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/conda_env_yaml.md b/src/content/tools/docs/1.14/pipeline_lint_tests/conda_env_yaml.md new file mode 100644 index 0000000000..f2dab4fba1 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/conda_env_yaml.md @@ -0,0 +1,36 @@ +# conda_env_yaml + +#### `PipelineLint.conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +:::note +This test is ignored if there is not an `environment.yml` +file present in the pipeline root directory. +::: + +DSL1 nf-core pipelines use a single Conda environment to manage all software +dependencies for a workflow. This can be used directly with `-profile conda` +and is also used in the `Dockerfile` to build a docker image. + +This test checks the conda `environment.yml` file to ensure that it follows nf-core guidelines. +Each dependency is checked using the [Anaconda API service](https://api.anaconda.org/docs). +Dependency sublists are ignored with the exception of `- pip`: these packages are also checked +for pinned version numbers and checked using the [PyPI JSON API](https://wiki.python.org/moin/PyPIJSON). + +Specifically, this lint test makes sure that: + +- The environment `name` must match the pipeline name and version + > - The pipeline name is defined in the config variable `manifest.name` + > - Replace the slash with a hyphen as environment names shouldn’t contain that character + > - Example: For `nf-core/test` version 1.4, the conda environment name should be `nf-core-test-1.4` +- All package dependencies have a specific version number pinned + > :::warning + > Remember that Conda package versions should be pinned with one equals sign (`toolname=1.1`), + > but pip uses two (`toolname==1.2`) + > ::: +- That package versions can be found and are the latest available + > - Test will go through all conda channels listed in the file, or check PyPI if `pip` + > - Conda dependencies with pinned channels (eg. `conda-forge::openjdk`) are ok too + > - In addition to the package name, the pinned version is checked + > - If a newer version is available, a warning will be reported diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/1.14/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..14c85336d8 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/files_exist.md @@ -0,0 +1,77 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/markdownlint.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/markdown_to_html.py +CHANGELOG.md +CODE_OF_CONDUCT.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +environment.yml +Dockerfile +conf/base.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/1.14/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ea06ada29f --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +'.gitattributes', +'.github/.dockstore.yml', +'.github/CONTRIBUTING.md', +'.github/ISSUE_TEMPLATE/bug_report.md', +'.github/ISSUE_TEMPLATE/config.yml', +'.github/ISSUE_TEMPLATE/feature_request.md', +'.github/markdownlint.yml', +'.github/PULL_REQUEST_TEMPLATE.md', +'.github/workflows/branch.yml', +'.github/workflows/linting_comment.yml', +'.github/workflows/linting.yml', +'assets/email_template.html', +'assets/email_template.txt', +'assets/nf-core-PIPELINE_logo.png', +'assets/sendmail_template.txt', +'bin/markdown_to_html.py', +'CODE_OF_CONDUCT.md', +'docs/images/nf-core-PIPELINE_logo.png', +'docs/README.md', +'lib/nfcore_external_java_deps.jar' +'lib/NfcoreSchema.groovy', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +'.github/workflows/push_dockerhub_dev.yml', +'.github/workflows/push_dockerhub_release.yml', +'.gitignore', +'assets/multiqc_config.yaml', +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/index.md b/src/content/tools/docs/1.14/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..0dc9aca3b7 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/index.md @@ -0,0 +1,42 @@ +# Pipline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [conda_dockerfile](conda_dockerfile) + - [`PipelineLint.conda_dockerfile()`](conda_dockerfile#nf_core.lint.PipelineLint.conda_dockerfile) +- [conda_env_yaml](conda_env_yaml) + - [`PipelineLint.conda_env_yaml()`](conda_env_yaml#nf_core.lint.PipelineLint.conda_env_yaml) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [params_used](params_used) + - [`PipelineLint.params_used()`](params_used#nf_core.lint.PipelineLint.params_used) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/1.14/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/1.14/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..8d911801f5 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,96 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) +- `process.container` + - Docker Hub handle for a single default container for use by all processes. + - Must specify a tag that matches the pipeline version number if set. + - If the pipeline version number contains the string `dev`, the DockerHub tag must be `:dev` + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/params_used.md b/src/content/tools/docs/1.14/pipeline_lint_tests/params_used.md new file mode 100644 index 0000000000..7290540d02 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/params_used.md @@ -0,0 +1,5 @@ +# params_used + +#### `PipelineLint.params_used(){:python}` + +Check for that params in `nextflow.config` are mentioned in `main.nf`. diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/1.14/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/1.14/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/readme.md b/src/content/tools/docs/1.14/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/1.14/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/1.14/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/1.14/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/1.14/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/1.14/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/1.14/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/1.14/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/1.5/bump_version.md b/src/content/tools/docs/1.5/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.5/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.5/create.md b/src/content/tools/docs/1.5/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.5/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.5/download.md b/src/content/tools/docs/1.5/download.md new file mode 100644 index 0000000000..79f40d0d68 --- /dev/null +++ b/src/content/tools/docs/1.5/download.md @@ -0,0 +1,56 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None){:python}` + +Downloads a nf-core workflow from Github to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `download_wf_files(){:python}` + +Downloads workflow files from Github to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – diff --git a/src/content/tools/docs/1.5/index.md b/src/content/tools/docs/1.5/index.md new file mode 100644 index 0000000000..fd6054924b --- /dev/null +++ b/src/content/tools/docs/1.5/index.md @@ -0,0 +1,37 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.5/licences.md b/src/content/tools/docs/1.5/licences.md new file mode 100644 index 0000000000..731a506571 --- /dev/null +++ b/src/content/tools/docs/1.5/licences.md @@ -0,0 +1,39 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `print_licences(as_json=False){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. diff --git a/src/content/tools/docs/1.5/lint.md b/src/content/tools/docs/1.5/lint.md new file mode 100644 index 0000000000..da9bb3b4f6 --- /dev/null +++ b/src/content/tools/docs/1.5/lint.md @@ -0,0 +1,300 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.singleEnd = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_ci_config(){:python}` + +Checks that the Travis or Circle CI YAML config is valid. + +Makes sure that `nf-core lint` runs in travis tests and that +tests run with the required nextflow version. + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'Dockerfile', +['.travis.yml', '.circle.yml'], +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container`, `process.container` and `$TRAVIS_TAG` (each only if set). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.5/list.md b/src/content/tools/docs/1.5/list.md new file mode 100644 index 0000000000..432af5d7a2 --- /dev/null +++ b/src/content/tools/docs/1.5/list.md @@ -0,0 +1,89 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the Github repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release'){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](http://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.5/utils.md b/src/content/tools/docs/1.5/utils.md new file mode 100644 index 0000000000..340ec4292f --- /dev/null +++ b/src/content/tools/docs/1.5/utils.md @@ -0,0 +1,22 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.fetch_wf_config(wf_path, wf=None){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be generated by tempfile.gettempdir() under +a nfcore_cache subdir. diff --git a/src/content/tools/docs/1.5/workflow.md b/src/content/tools/docs/1.5/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.5/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.6/bump_version.md b/src/content/tools/docs/1.6/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.6/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.6/create.md b/src/content/tools/docs/1.6/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.6/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.6/download.md b/src/content/tools/docs/1.6/download.md new file mode 100644 index 0000000000..79f40d0d68 --- /dev/null +++ b/src/content/tools/docs/1.6/download.md @@ -0,0 +1,56 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None){:python}` + +Downloads a nf-core workflow from Github to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `download_wf_files(){:python}` + +Downloads workflow files from Github to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – diff --git a/src/content/tools/docs/1.6/index.md b/src/content/tools/docs/1.6/index.md new file mode 100644 index 0000000000..fd6054924b --- /dev/null +++ b/src/content/tools/docs/1.6/index.md @@ -0,0 +1,37 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.6/licences.md b/src/content/tools/docs/1.6/licences.md new file mode 100644 index 0000000000..731a506571 --- /dev/null +++ b/src/content/tools/docs/1.6/licences.md @@ -0,0 +1,39 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `print_licences(as_json=False){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. diff --git a/src/content/tools/docs/1.6/lint.md b/src/content/tools/docs/1.6/lint.md new file mode 100644 index 0000000000..da9bb3b4f6 --- /dev/null +++ b/src/content/tools/docs/1.6/lint.md @@ -0,0 +1,300 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.singleEnd = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_ci_config(){:python}` + +Checks that the Travis or Circle CI YAML config is valid. + +Makes sure that `nf-core lint` runs in travis tests and that +tests run with the required nextflow version. + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'Dockerfile', +['.travis.yml', '.circle.yml'], +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container`, `process.container` and `$TRAVIS_TAG` (each only if set). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.6/list.md b/src/content/tools/docs/1.6/list.md new file mode 100644 index 0000000000..432af5d7a2 --- /dev/null +++ b/src/content/tools/docs/1.6/list.md @@ -0,0 +1,89 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the Github repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release'){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](http://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.6/utils.md b/src/content/tools/docs/1.6/utils.md new file mode 100644 index 0000000000..340ec4292f --- /dev/null +++ b/src/content/tools/docs/1.6/utils.md @@ -0,0 +1,22 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.fetch_wf_config(wf_path, wf=None){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be generated by tempfile.gettempdir() under +a nfcore_cache subdir. diff --git a/src/content/tools/docs/1.6/workflow.md b/src/content/tools/docs/1.6/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.6/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.7/bump_version.md b/src/content/tools/docs/1.7/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.7/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.7/create.md b/src/content/tools/docs/1.7/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.7/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.7/download.md b/src/content/tools/docs/1.7/download.md new file mode 100644 index 0000000000..c9d784edac --- /dev/null +++ b/src/content/tools/docs/1.7/download.md @@ -0,0 +1,68 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Downloads a nf-core workflow from Github to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from Github to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.7/index.md b/src/content/tools/docs/1.7/index.md new file mode 100644 index 0000000000..fd6054924b --- /dev/null +++ b/src/content/tools/docs/1.7/index.md @@ -0,0 +1,37 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.7/licences.md b/src/content/tools/docs/1.7/licences.md new file mode 100644 index 0000000000..731a506571 --- /dev/null +++ b/src/content/tools/docs/1.7/licences.md @@ -0,0 +1,39 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `print_licences(as_json=False){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. diff --git a/src/content/tools/docs/1.7/lint.md b/src/content/tools/docs/1.7/lint.md new file mode 100644 index 0000000000..da9bb3b4f6 --- /dev/null +++ b/src/content/tools/docs/1.7/lint.md @@ -0,0 +1,300 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.singleEnd = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_ci_config(){:python}` + +Checks that the Travis or Circle CI YAML config is valid. + +Makes sure that `nf-core lint` runs in travis tests and that +tests run with the required nextflow version. + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'Dockerfile', +['.travis.yml', '.circle.yml'], +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container`, `process.container` and `$TRAVIS_TAG` (each only if set). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.7/list.md b/src/content/tools/docs/1.7/list.md new file mode 100644 index 0000000000..432af5d7a2 --- /dev/null +++ b/src/content/tools/docs/1.7/list.md @@ -0,0 +1,89 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the Github repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release'){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](http://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.7/utils.md b/src/content/tools/docs/1.7/utils.md new file mode 100644 index 0000000000..48285a4fe6 --- /dev/null +++ b/src/content/tools/docs/1.7/utils.md @@ -0,0 +1,22 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.fetch_wf_config(wf_path, wf=None){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. diff --git a/src/content/tools/docs/1.7/workflow.md b/src/content/tools/docs/1.7/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.7/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.8/bump_version.md b/src/content/tools/docs/1.8/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.8/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.8/create.md b/src/content/tools/docs/1.8/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.8/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.8/download.md b/src/content/tools/docs/1.8/download.md new file mode 100644 index 0000000000..c9d784edac --- /dev/null +++ b/src/content/tools/docs/1.8/download.md @@ -0,0 +1,68 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Downloads a nf-core workflow from Github to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from Github to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.8/index.md b/src/content/tools/docs/1.8/index.md new file mode 100644 index 0000000000..fd6054924b --- /dev/null +++ b/src/content/tools/docs/1.8/index.md @@ -0,0 +1,37 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.8/licences.md b/src/content/tools/docs/1.8/licences.md new file mode 100644 index 0000000000..731a506571 --- /dev/null +++ b/src/content/tools/docs/1.8/licences.md @@ -0,0 +1,39 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `print_licences(as_json=False){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. diff --git a/src/content/tools/docs/1.8/lint.md b/src/content/tools/docs/1.8/lint.md new file mode 100644 index 0000000000..5b8c83cc64 --- /dev/null +++ b/src/content/tools/docs/1.8/lint.md @@ -0,0 +1,336 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub actions ci workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_ci_config(){:python}` + +Checks that the Travis or Circle CI YAML config is valid. + +Makes sure that `nf-core lint` runs in travis tests and that +tests run with the required nextflow version. + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'Dockerfile', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workfows/linting.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container`, `process.container` and `$TRAVIS_TAG` (each only if set). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.8/list.md b/src/content/tools/docs/1.8/list.md new file mode 100644 index 0000000000..432af5d7a2 --- /dev/null +++ b/src/content/tools/docs/1.8/list.md @@ -0,0 +1,89 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the Github repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release'){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](http://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.8/utils.md b/src/content/tools/docs/1.8/utils.md new file mode 100644 index 0000000000..48285a4fe6 --- /dev/null +++ b/src/content/tools/docs/1.8/utils.md @@ -0,0 +1,22 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.fetch_wf_config(wf_path, wf=None){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. diff --git a/src/content/tools/docs/1.8/workflow.md b/src/content/tools/docs/1.8/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.8/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/1.9/bump_version.md b/src/content/tools/docs/1.9/bump_version.md new file mode 100644 index 0000000000..388370558e --- /dev/null +++ b/src/content/tools/docs/1.9/bump_version.md @@ -0,0 +1,36 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(lint_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(lint_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, lint_obj, pattern, newstr, allow_multiple=False){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **lint_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. + - **allow_multiple** (_bool_) – Replace all pattern hits, not only the first. Defaults to False. +- **Raises:** + **SyntaxError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/1.9/create.md b/src/content/tools/docs/1.9/create.md new file mode 100644 index 0000000000..354aadee7b --- /dev/null +++ b/src/content/tools/docs/1.9/create.md @@ -0,0 +1 @@ +# nf_core.create diff --git a/src/content/tools/docs/1.9/download.md b/src/content/tools/docs/1.9/download.md new file mode 100644 index 0000000000..fbea76fc98 --- /dev/null +++ b/src/content/tools/docs/1.9/download.md @@ -0,0 +1,68 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline, release=None, singularity=False, outdir=None, compress_type='tar.gz'){:python}` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `fetch_workflow_details(wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + **wfs** ([_nf_core.list.Workflows_](list#nf_core.list.Workflows)) – A nf_core.list.Workflows object +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +#### `find_container_images(){:python}` + +Find container image names for workflow + +#### `pull_singularity_image(container){:python}` + +Uses a local installation of singularity to pull an image from Docker Hub. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to nfcore/name:dev. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/1.9/index.md b/src/content/tools/docs/1.9/index.md new file mode 100644 index 0000000000..fd6054924b --- /dev/null +++ b/src/content/tools/docs/1.9/index.md @@ -0,0 +1,37 @@ + + +# Welcome to nf-core tools API documentation! + +# Contents: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) +- [nf_core.download](download) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) + - [`run_linting()`](lint#nf_core.lint.run_linting) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.utils](utils) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) +- [nf_core.list](workflow) + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/1.9/licences.md b/src/content/tools/docs/1.9/licences.md new file mode 100644 index 0000000000..731a506571 --- /dev/null +++ b/src/content/tools/docs/1.9/licences.md @@ -0,0 +1,39 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `clean_licence_names(licences){:python}` + +Normalises varying licence names. + +- **Parameters:** + **licences** (_list_) – A list of licences which are basically raw string objects from + the licence content information. +- **Returns:** + Cleaned licences. +- **Return type:** + list + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `print_licences(as_json=False){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. diff --git a/src/content/tools/docs/1.9/lint.md b/src/content/tools/docs/1.9/lint.md new file mode 100644 index 0000000000..5bf982b0d4 --- /dev/null +++ b/src/content/tools/docs/1.9/lint.md @@ -0,0 +1,332 @@ +# nf_core.lint + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### _`class{:python}`_`nf_core.lint.PipelineLint(path){:python}` + +Object to hold linting information and results. +All objects attributes are set, after the [`PipelineLint.lint_pipeline()`](#nf_core.lint.PipelineLint.lint_pipeline) function was called. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (environment.yml). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `dockerfile{:python}` + +A list of lines (str) from the parsed Dockerfile. + +- **Type:** + list + +#### `failed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `passed{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +#### `path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: (, ) + +- **Type:** + list + +**Attribute specifications** + +Some of the more complex attributes of a PipelineLint object. + +- conda_config: + ```default + # Example + { + 'name': 'nf-core-hlatyping', + 'channels': ['bioconda', 'conda-forge'], + 'dependencies': ['optitype=1.3.2', 'yara=0.9.6'] + } + ``` +- conda_package_info: + ```default + # See https://api.anaconda.org/package/bioconda/bioconda-utils as an example. + { + : + } + ``` +- config: Produced by calling Nextflow with `nextflow config -flat `. Here is an example from + : the [nf-core/hlatyping](https://github.com/nf-core/hlatyping) pipeline: + ```default + process.container = 'nfcore/hlatyping:1.1.1' + params.help = false + params.outdir = './results' + params.bam = false + params.single_end = false + params.seqtype = 'dna' + params.solver = 'glpk' + params.igenomes_base = './iGenomes' + params.clusterOptions = false + ... + ``` + +#### `check_actions_branch_protection(){:python}` + +Checks that the GitHub Actions branch protection workflow is valid. + +Makes sure PRs can only come from nf-core dev or ‘patch’ of a fork. + +#### `check_actions_ci(){:python}` + +Checks that the GitHub Actions CI workflow is valid + +Makes sure tests run with the required nextflow version. + +#### `check_actions_lint(){:python}` + +Checks that the GitHub Actions lint workflow is valid + +Makes sure `nf-core lint` and `markdownlint` runs. + +#### `check_anaconda_package(dep){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + **dep** (_str_) – A conda package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved.** – + +#### `check_conda_dockerfile(){:python}` + +Checks the Docker build file. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- dependency versions are pinned +- dependency versions are the latest available + +#### `check_conda_env_yaml(){:python}` + +Checks that the conda environment file is valid. + +Checks that: +: \* a name is given and is consistent with the pipeline name + +- check that dependency versions are pinned +- dependency versions are the latest available + +#### `check_docker(){:python}` + +Checks that Dockerfile contains the string `FROM`. + +#### `check_files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checkmarks files +for presence. +Files that **must** be present: + +```default +'nextflow.config', +'Dockerfile', +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +'README.md', +'CHANGELOG.md', +'docs/README.md', +'docs/output.md', +'docs/usage.md' +``` + +Files that _should_ be present: + +```default +'main.nf', +'environment.yml', +'conf/base.config', +'.github/workflows/branch.yml', +'.github/workflows/ci.yml', +'.github/workfows/linting.yml' +``` + +Files that _must not_ be present: + +```default +'Singularity' +``` + +Files that _should not_ be present: + +```default +'.travis.yml' +``` + +- **Raises:** + **An AssertionError if neither nextflow.config** **or** **main.nf found.** – + +#### `check_licence(){:python}` + +Checks licence file is MIT. + +Currently the checkpoints are: +: \* licence file must be long enough (4 or more lines) + +- licence contains the string _without restriction_ +- licence doesn’t have any placeholder variables + +#### `check_nextflow_config(){:python}` + +Checks a given pipeline for required config variables. + +At least one string in each list must be present for fail and warn. +Any config in config_fail_ifdefined results in a failure. + +Uses `nextflow config -flat` to parse pipeline `nextflow.config` +and print all config variables. +NB: Does NOT parse contents of main.nf / nextflow script + +#### `check_pip_package(dep){:python}` + +Query PyPi package information. + +Sends a HTTP GET request to the PyPi remote API. + +- **Parameters:** + **dep** (_str_) – A PyPi package name. +- **Raises:** + **A ValueError**\*\*,\*\* **if the package name can not be resolved** **or** **the connection timed out.** – + +#### `check_pipeline_name(){:python}` + +Check whether pipeline name adheres to lower case/no hyphen naming convention + +#### `check_pipeline_todos(){:python}` + +Go through all template files looking for the string ‘TODO nf-core:’ + +#### `check_readme(){:python}` + +Checks the repository README file for errors. + +Currently just checks the badges at the top of the README. + +#### `check_version_consistency(){:python}` + +Checks container tags versions. + +Runs on `process.container` (if set) and `$GITHUB_REF` (if a GitHub Actions release). + +Checks that: +: \* the container has a tag + +- the version numbers are numeric +- the version numbers are the same as one-another + +#### `lint_pipeline(release_mode=False){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +and returns summary at completion. Raises an exception if there is a +critical error that makes the rest of the tests pointless (eg. no +pipeline script). Results from this function are printed by the main script. + +- **Parameters:** + **release_mode** (_boolean_) – Activates the release mode, which checks for + consistent version tags of containers. Default is False. +- **Returns:** + Summary of test result messages structured as follows: + ```default + { + 'pass': [ + ( test-id (int), message (string) ), + ( test-id (int), message (string) ) + ], + 'warn': [(id, msg)], + 'fail': [(id, msg)], + } + ``` +- **Return type:** + dict +- **Raises:** + **If a critical problem is found**\*\*,\*\* **an AssertionError is raised.** – + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. diff --git a/src/content/tools/docs/1.9/list.md b/src/content/tools/docs/1.9/list.md new file mode 100644 index 0000000000..138558a207 --- /dev/null +++ b/src/content/tools/docs/1.9/list.md @@ -0,0 +1,89 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release'){:python}` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](http://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/1.9/utils.md b/src/content/tools/docs/1.9/utils.md new file mode 100644 index 0000000000..48285a4fe6 --- /dev/null +++ b/src/content/tools/docs/1.9/utils.md @@ -0,0 +1,22 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### `nf_core.utils.fetch_wf_config(wf_path, wf=None){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. diff --git a/src/content/tools/docs/1.9/workflow.md b/src/content/tools/docs/1.9/workflow.md new file mode 100644 index 0000000000..9e44b801a9 --- /dev/null +++ b/src/content/tools/docs/1.9/workflow.md @@ -0,0 +1 @@ +# nf_core.list diff --git a/src/content/tools/docs/2.0.1/api/bump_version.md b/src/content/tools/docs/2.0.1/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.0.1/api/create.md b/src/content/tools/docs/2.0.1/api/create.md new file mode 100644 index 0000000000..302bfd36a6 --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.0.1/api/download.md b/src/content/tools/docs/2.0.1/api/download.md new file mode 100644 index 0000000000..7769825e08 --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/download.md @@ -0,0 +1,142 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, release=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_release_hash(){:python}` + +Find specified release / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_release(){:python}` + +Prompt for pipeline release / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.0.1/api/index.md b/src/content/tools/docs/2.0.1/api/index.md new file mode 100644 index 0000000000..958932acef --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/index.md @@ -0,0 +1,55 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.0.1/api/launch.md b/src/content/tools/docs/2.0.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.0.1/api/licences.md b/src/content/tools/docs/2.0.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.0.1/api/lint.md b/src/content/tools/docs/2.0.1/api/lint.md new file mode 100644 index 0000000000..58da46f58a --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.0.1/api/list.md b/src/content/tools/docs/2.0.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.0.1/api/modules.md b/src/content/tools/docs/2.0.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.0.1/api/schema.md b/src/content/tools/docs/2.0.1/api/schema.md new file mode 100644 index 0000000000..bbb2b2e39b --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/schema.md @@ -0,0 +1,113 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.0.1/api/sync.md b/src/content/tools/docs/2.0.1/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.0.1/api/utils.md b/src/content/tools/docs/2.0.1/api/utils.md new file mode 100644 index 0000000000..1895d7da6f --- /dev/null +++ b/src/content/tools/docs/2.0.1/api/utils.md @@ -0,0 +1,264 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.0.1/index.md b/src/content/tools/docs/2.0.1/index.md new file mode 100644 index 0000000000..c3d07a7628 --- /dev/null +++ b/src/content/tools/docs/2.0.1/index.md @@ -0,0 +1,13 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..e4e5cb0a57 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,85 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/scrape_software_versions.py +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +modules/local/get_software_versions.nf +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yaml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..9b7e597ec6 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,55 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +assets/multiqc_config.yaml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..d73f5fbd15 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/index.md @@ -0,0 +1,38 @@ +# Pipline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..40c566e813 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version git_sha hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.0.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.0.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.0.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.0/api/bump_version.md b/src/content/tools/docs/2.0/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.0/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.0/api/create.md b/src/content/tools/docs/2.0/api/create.md new file mode 100644 index 0000000000..302bfd36a6 --- /dev/null +++ b/src/content/tools/docs/2.0/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.0/api/download.md b/src/content/tools/docs/2.0/api/download.md new file mode 100644 index 0000000000..7769825e08 --- /dev/null +++ b/src/content/tools/docs/2.0/api/download.md @@ -0,0 +1,142 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, release=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **release** (_str_) – The workflow release version to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_release_hash(){:python}` + +Find specified release / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_release(){:python}` + +Prompt for pipeline release / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.0/api/index.md b/src/content/tools/docs/2.0/api/index.md new file mode 100644 index 0000000000..958932acef --- /dev/null +++ b/src/content/tools/docs/2.0/api/index.md @@ -0,0 +1,55 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.0/api/launch.md b/src/content/tools/docs/2.0/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.0/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.0/api/licences.md b/src/content/tools/docs/2.0/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.0/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.0/api/lint.md b/src/content/tools/docs/2.0/api/lint.md new file mode 100644 index 0000000000..58da46f58a --- /dev/null +++ b/src/content/tools/docs/2.0/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.0/api/list.md b/src/content/tools/docs/2.0/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.0/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.0/api/modules.md b/src/content/tools/docs/2.0/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.0/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.0/api/schema.md b/src/content/tools/docs/2.0/api/schema.md new file mode 100644 index 0000000000..bbb2b2e39b --- /dev/null +++ b/src/content/tools/docs/2.0/api/schema.md @@ -0,0 +1,113 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.0/api/sync.md b/src/content/tools/docs/2.0/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.0/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.0/api/utils.md b/src/content/tools/docs/2.0/api/utils.md new file mode 100644 index 0000000000..1895d7da6f --- /dev/null +++ b/src/content/tools/docs/2.0/api/utils.md @@ -0,0 +1,264 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.0/index.md b/src/content/tools/docs/2.0/index.md new file mode 100644 index 0000000000..c3d07a7628 --- /dev/null +++ b/src/content/tools/docs/2.0/index.md @@ -0,0 +1,13 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.0/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..e4e5cb0a57 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/files_exist.md @@ -0,0 +1,85 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/scrape_software_versions.py +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +modules/local/get_software_versions.nf +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yaml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.0/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..9b7e597ec6 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,55 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +assets/multiqc_config.yaml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/index.md b/src/content/tools/docs/2.0/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..d73f5fbd15 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/index.md @@ -0,0 +1,38 @@ +# Pipline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.0/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.0/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..40c566e813 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version git_sha hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.0/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.0/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.0/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.0/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.0/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.0/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.0/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.0/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.0/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.0/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.0/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.1/api/bump_version.md b/src/content/tools/docs/2.1/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.1/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.1/api/create.md b/src/content/tools/docs/2.1/api/create.md new file mode 100644 index 0000000000..302bfd36a6 --- /dev/null +++ b/src/content/tools/docs/2.1/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.1/api/download.md b/src/content/tools/docs/2.1/api/download.md new file mode 100644 index 0000000000..a04af4c422 --- /dev/null +++ b/src/content/tools/docs/2.1/api/download.md @@ -0,0 +1,142 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.1/api/index.md b/src/content/tools/docs/2.1/api/index.md new file mode 100644 index 0000000000..8f79de1474 --- /dev/null +++ b/src/content/tools/docs/2.1/api/index.md @@ -0,0 +1,56 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.1/api/launch.md b/src/content/tools/docs/2.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.1/api/licences.md b/src/content/tools/docs/2.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.1/api/lint.md b/src/content/tools/docs/2.1/api/lint.md new file mode 100644 index 0000000000..58da46f58a --- /dev/null +++ b/src/content/tools/docs/2.1/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.1/api/list.md b/src/content/tools/docs/2.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.1/api/modules.md b/src/content/tools/docs/2.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.1/api/schema.md b/src/content/tools/docs/2.1/api/schema.md new file mode 100644 index 0000000000..bbb2b2e39b --- /dev/null +++ b/src/content/tools/docs/2.1/api/schema.md @@ -0,0 +1,113 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.1/api/sync.md b/src/content/tools/docs/2.1/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.1/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.1/api/utils.md b/src/content/tools/docs/2.1/api/utils.md new file mode 100644 index 0000000000..d2c70d3ad7 --- /dev/null +++ b/src/content/tools/docs/2.1/api/utils.md @@ -0,0 +1,268 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + **wf_path** (_str_) – Nextflow workflow file system path. +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.1/index.md b/src/content/tools/docs/2.1/index.md new file mode 100644 index 0000000000..c3d07a7628 --- /dev/null +++ b/src/content/tools/docs/2.1/index.md @@ -0,0 +1,13 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..e4e5cb0a57 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,85 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +bin/scrape_software_versions.py +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +modules/local/get_software_versions.nf +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yaml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..9b7e597ec6 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,55 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.md +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +assets/multiqc_config.yaml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..d73f5fbd15 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/index.md @@ -0,0 +1,38 @@ +# Pipline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..40c566e813 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version git_sha hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.10/api/bump_version.md b/src/content/tools/docs/2.10/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.10/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.10/api/create.md b/src/content/tools/docs/2.10/api/create.md new file mode 100644 index 0000000000..756f822c77 --- /dev/null +++ b/src/content/tools/docs/2.10/api/create.md @@ -0,0 +1,88 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain, pipeline_dir){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. + - **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + - **plain** (_bool_) – If true the pipeline template will be initialized plain, without customisation. + - **pipeline_dir** (_str_) – Path to the pipeline directory. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + **template_areas** (_list_) – List of available template areas to skip. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.10/api/download.md b/src/content/tools/docs/2.10/api/download.md new file mode 100644 index 0000000000..e2015e642b --- /dev/null +++ b/src/content/tools/docs/2.10/api/download.md @@ -0,0 +1,309 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`exception{:python}`_`nf_core.download.ContainerError(container, registry, address, absolute_URI, out_path, singularity_command, error_msg){:python}` + +Bases: `Exception` + +A class of errors related to pulling containers with Singularity/Apptainer + +#### `undefined{:python}` ImageExists(error_log) + +Bases: `FileExistsError` + +Image already exists in cache/output directory. + +#### `undefined{:python}` ImageNotFound(error_log) + +Bases: `FileNotFoundError` + +The image can not be found in the registry + +#### `undefined{:python}` InvalidTag(error_log) + +Bases: `AttributeError` + +Image and registry are valid, but the (version) tag is not + +#### `undefined{:python}` OtherError(error_log) + +Bases: `RuntimeError` + +Undefined error with the container + +#### `undefined{:python}` RegistryNotFound(error_log) + +Bases: `ConnectionRefusedError` + +The specified registry does not resolve to a valid IP address + +### _`exception{:python}`_`nf_core.download.DownloadError{:python}` + +Bases: `RuntimeError` + +A custom exception that is raised when nf-core download encounters a problem that we already took into consideration. +In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, tower=False, download_configuration=None, container_system=None, container_library=None, container_cache_utilisation=None, container_cache_index=None, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_List_\*\[**str**]\*) – The workflow revision to download, like 1.0. Defaults to None. + - **container** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **tower** (_bool_) – Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(revision, wf_sha, download_url){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `download_workflow_static(){:python}` + +Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner. + +#### `download_workflow_tower(location=None){:python}` + +Create a bare-cloned git repository of the workflow, so it can be launched with tw launch as pipeline + +#### `find_container_images(workflow_directory){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic, +but not always, e.g. not for differentialabundance 1.2.0 + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw matches that will likely need to be cleaned. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(current_revision=''){:python}` + +Loop through container names and download Singularity images + +#### `prioritize_direct_download(container_list){:python}` + +Helper function that takes a list of container images (URLs and Docker URIs), +eliminates all Docker URIs for which also a URL is contained and returns the +cleaned and also deduplicated list. + +Conceptually, this works like so: + +Everything after the last Slash should be identical, e.g. “scanpy:1.7.2–pyhdfd78af_0” in +\[‘https://depot.galaxyproject.org/singularity/scanpy:1.7.2–pyhdfd78af\_0’, ‘biocontainers/scanpy:1.7.2–pyhdfd78af_0’] + +re.sub(‘.\*/(.\*)’,’1’,c) will drop everything up to the last slash from c (container_id) + +d.get(k:=re.sub(‘.\*/(.\*)’,’1’,c),’’) assigns the truncated string to k (key) and gets the +corresponding value from the dict if present or else defaults to “”. + +If the regex pattern matches, the original container_id will be assigned to the dict with the k key. +r”^$|(?!^http)” matches an empty string (we didn’t have it in the dict yet and want to keep it in either case) or +any string that does not start with http. Because if our current dict value already starts with http, +we want to keep it and not replace with with whatever we have now (which might be the Docker URI). + +A regex that matches http, r”^$|^http” could thus be used to prioritize the Docker URIs over http Downloads + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_config_inclusion(){:python}` + +Prompt for inclusion of institutional configurations + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch +Prompt user for revision tag if ‘–revision’ was not set +If –tower is specified, allow to select multiple revisions +Also the static download allows for multiple revisions, but +we do not prompt this option interactively. + +#### `prompt_singularity_cachedir_creation(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `prompt_singularity_cachedir_remote(){:python}` + +Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR + +#### `prompt_singularity_cachedir_utilization(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `read_remote_containers(){:python}` + +Reads the file specified as index for the remote Singularity cache dir + +#### `rectify_raw_container_matches(raw_findings){:python}` + +Helper function to rectify the raw extracted container matches into fully qualified container names. +If multiple containers are found, any prefixed with http for direct download is prioritized + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘biocontainers/fastqc:0.11.9–0’ }” + +Later DSL2, variable is being used: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity\_pull\_docker\_container ? +: “/${container_id}” : +“quay.io/biocontainers/${container_id}” }”
      +container_id = ‘mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0’ + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, library, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. + - **library** (_list_ _of_ _str_) – A list of libraries to try for pulling the image. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(revision_dirname){:python}` + +Edit the downloaded nextflow.config file to use the local config files + +### _`class{:python}`_`nf_core.download.WorkflowRepo(remote_url, revision, commit, location=None, hide_progress=False, in_cache=True){:python}` + +Bases: `SyncedRepo` + +An object to store details about a locally cached workflow repository. + +Important Attributes: +: fullname: The full name of the repository, `nf-core/{self.pipelinename}`. +local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to `$HOME/.cache/nf-core/nf-core/{self.pipeline}`. + +#### `access(){:python}` + +#### `bare_clone(destination){:python}` + +#### `checkout(commit){:python}` + +Checks out the repository at the requested commit + +- **Parameters:** + **commit** (_str_) – Git SHA of the commit + +#### `get_remote_branches(remote_url){:python}` + +Get all branches from a remote repository + +- **Parameters:** + **remote_url** (_str_) – The git url to the remote repository +- **Returns:** + All branches found in the remote +- **Return type:** + (set\[str]) + +#### `retry_setup_local_repo(skip_confirm=False){:python}` + +#### `setup_local_repo(remote, location=None, in_cache=True){:python}` + +Sets up the local git repository. If the repository has been cloned previously, it +returns a git.Repo object of that clone. Otherwise it tries to clone the repository from +the provided remote URL and returns a git.Repo of the new clone. + +- **Parameters:** + - **remote** (_str_) – git url of remote + - **location** (_Path_) – location where the clone should be created/cached. + - **in_cache** (_bool_\*,\* _optional_) – Whether to clone the repository from the cache. Defaults to False. + +Sets self.repo + +#### `tidy_tags_and_branches(){:python}` + +Function to delete all tags and branches that are not of interest to the downloader. +This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + +However, due to local caching, the downloader might also want access to revisions that had been deleted before. +In that case, don’t bother with re-adding the tags and rather download anew from Github. diff --git a/src/content/tools/docs/2.10/api/index.md b/src/content/tools/docs/2.10/api/index.md new file mode 100644 index 0000000000..41c03f0fa1 --- /dev/null +++ b/src/content/tools/docs/2.10/api/index.md @@ -0,0 +1,75 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`ContainerError`](download#nf_core.download.ContainerError) + - [`DownloadError`](download#nf_core.download.DownloadError) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) + - [`WorkflowRepo`](download#nf_core.download.WorkflowRepo) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.params_file](params-file) + - [`ParamsFileBuilder`](params-file#nf_core.params_file.ParamsFileBuilder) + - [`_print_wrapped()`](params-file#nf_core.params_file._print_wrapped) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`SingularityCacheFilePathValidator`](utils#nf_core.utils.SingularityCacheFilePathValidator) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_remote_version()`](utils#nf_core.utils.fetch_remote_version) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.10/api/launch.md b/src/content/tools/docs/2.10/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.10/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.10/api/licences.md b/src/content/tools/docs/2.10/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.10/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.10/api/lint.md b/src/content/tools/docs/2.10/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.10/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.10/api/list.md b/src/content/tools/docs/2.10/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.10/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.10/api/modules.md b/src/content/tools/docs/2.10/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.10/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.10/api/params-file.md b/src/content/tools/docs/2.10/api/params-file.md new file mode 100644 index 0000000000..6cef34b10a --- /dev/null +++ b/src/content/tools/docs/2.10/api/params-file.md @@ -0,0 +1,84 @@ +# nf_core.params_file + +Create a YAML parameter file + +### _`class{:python}`_`nf_core.params_file.ParamsFileBuilder(pipeline=None, revision=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline. + +- **Parameters:** + - **pipeline** (_str_\*,\* _optional_) – Path to a local pipeline path or a remote pipeline. + - **revision** (_str_\*,\* _optional_) – Revision of the pipeline to use. + +#### `format_group(definition, show_hidden=False){:python}` + +Format a group of parameters of the schema as commented YAML. + +- **Parameters:** + - **definition** (_dict_) – Definition of the group from the schema + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for a group +- **Return type:** + str + +#### `format_param(name, properties, required_properties=(), show_hidden=False){:python}` + +Format a single parameter of the schema as commented YAML + +- **Parameters:** + - **name** (_str_) – Name of the parameter + - **properties** (_dict_) – Properties of the parameter + - **required_properties** (_list_) – List of required properties + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Section of a params-file.yml for given parameter + None: + > If the parameter is skipped because it is hidden and + > show_hidden is not set +- **Return type:** + str + +#### `generate_params_file(show_hidden=False){:python}` + +Generate the contents of a parameter template file. + +Assumes the pipeline has been fetched (if remote) and the schema loaded. + +- **Parameters:** + **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for the pipeline schema +- **Return type:** + str + +#### `get_pipeline(){:python}` + +Prompt the user for a pipeline name and get the schema + +#### `write_params_file(output_fn='nf-params.yaml', show_hidden=False, force=False){:python}` + +Build a template file for the pipeline schema. + +- **Parameters:** + - **output_fn** (_str_\*,\* _optional_) – Filename to write the template to. + - **show_hidden** (_bool_\*,\* _optional_) – Include parameters marked as hidden in the output + - **force** (_bool_\*,\* _optional_) – Whether to overwrite existing output file. +- **Returns:** + True if the template was written successfully, False otherwise +- **Return type:** + bool + +### `nf_core.params_file._print_wrapped(text, fill_char='-', mode='both', width=80, indent=0, drop_whitespace=True){:python}` + +Helper function to format text for the params-file template. + +- **Parameters:** + - **text** (_str_) – Text to print + - **fill_char** (_str_\*,\* _optional_) – Character to use for creating dividers. Defaults to ‘-‘. + - **mode** (_str_\*,\* _optional_) – Where to place dividers. Defaults to “both”. + - **width** (_int_\*,\* _optional_) – Maximum line-width of the output text. Defaults to 80. + - **indent** (_int_\*,\* _optional_) – Number of spaces to indent the text. Defaults to 0. + - **drop_whitespace** (_bool_\*,\* _optional_) – Whether to drop whitespace from the start and end of lines. diff --git a/src/content/tools/docs/2.10/api/schema.md b/src/content/tools/docs/2.10/api/schema.md new file mode 100644 index 0000000000..d795ff7452 --- /dev/null +++ b/src/content/tools/docs/2.10/api/schema.md @@ -0,0 +1,165 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.10/api/subworkflows.md b/src/content/tools/docs/2.10/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.10/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.10/api/sync.md b/src/content/tools/docs/2.10/api/sync.md new file mode 100644 index 0000000000..63da8cf89f --- /dev/null +++ b/src/content/tools/docs/2.10/api/sync.md @@ -0,0 +1,152 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. DEPRECATED + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.10/api/utils.md b/src/content/tools/docs/2.10/api/utils.md new file mode 100644 index 0000000000..ad00c3a5a8 --- /dev/null +++ b/src/content/tools/docs/2.10/api/utils.md @@ -0,0 +1,380 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### _`class{:python}`_`nf_core.utils.SingularityCacheFilePathValidator{:python}` + +Bases: `Validator` + +Validator for file path specified as –singularity-cache-index argument in nf-core download + +#### `_abc_impl{:python}`_= <\_abc.\_abc_data object>_ + +#### `validate(value){:python}` + +Validate the input. +If invalid, this should raise a `ValidationError`. + +- **Parameters:** + **document** – `Document` instance. + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_remote_version(source_url){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API + - **multiple** (_bool_) – Allow selection of multiple releases & branches (for Tower) +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.10/index.md b/src/content/tools/docs/2.10/index.md new file mode 100644 index 0000000000..636ffaa787 --- /dev/null +++ b/src/content/tools/docs/2.10/index.md @@ -0,0 +1,12 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [Subworkflow code lint tests](subworkflow_lint_tests/index) (run by `nf-core subworkflows lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.10/module_lint_tests/index.md b/src/content/tools/docs/2.10/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.10/module_lint_tests/main_nf.md b/src/content/tools/docs/2.10/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..aa42cef6d7 --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, registry, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.10/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.10/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..b1045c5fbd --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.10/module_lint_tests/module_changes.md b/src/content/tools/docs/2.10/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.10/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.10/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.10/module_lint_tests/module_patch.md b/src/content/tools/docs/2.10/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..5d38a40d31 --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreComponent){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.10/module_lint_tests/module_tests.md b/src/content/tools/docs/2.10/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..a2d227becf --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/module_tests.md @@ -0,0 +1,10 @@ +# module_tests + +#### `ModuleLint.module_tests(module){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the module is present in the `pytest_modules.yml` +file. diff --git a/src/content/tools/docs/2.10/module_lint_tests/module_todos.md b/src/content/tools/docs/2.10/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.10/module_lint_tests/module_version.md b/src/content/tools/docs/2.10/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.10/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..41e0a6fbef --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.10/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..d65e9b5d8c --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +pipeline_template.yml # saving information in .nf-core.yml +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.10/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..6247c86c45 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,56 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/index.md b/src/content/tools/docs/2.10/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..c8cf0fe7b1 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/index.md @@ -0,0 +1,44 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [system_exit](system_exit) + - [`PipelineLint.system_exit()`](system_exit#nf_core.lint.PipelineLint.system_exit) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.10/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.10/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.10/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.10/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.10/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..14c9d5322d --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,94 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.validationShowHiddenParams` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.validationSchemaIgnoreParams` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.10/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.10/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.10/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.10/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.10/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.10/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/2.10/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.10/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.10/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.10/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.10/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.10/subworkflow_lint_tests/index.md b/src/content/tools/docs/2.10/subworkflow_lint_tests/index.md new file mode 100644 index 0000000000..d5061028dc --- /dev/null +++ b/src/content/tools/docs/2.10/subworkflow_lint_tests/index.md @@ -0,0 +1,16 @@ +# Subworkflow lint tests + +# Tests: + +- [main_nf](main_nf) + - [`SubworkflowLint.main_nf()`](main_nf#nf_core.subworkflows.lint.SubworkflowLint.main_nf) +- [meta_yml](meta_yml) + - [`SubworkflowLint.meta_yml()`](meta_yml#nf_core.subworkflows.lint.SubworkflowLint.meta_yml) +- [subworkflow_changes](subworkflow_changes) + - [`SubworkflowLint.subworkflow_changes()`](subworkflow_changes#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_changes) +- [subworkflow_tests](subworkflow_tests) + - [`SubworkflowLint.subworkflow_tests()`](subworkflow_tests#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_tests) +- [subworkflow_todos](subworkflow_todos) + - [`SubworkflowLint.subworkflow_todos()`](subworkflow_todos#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_todos) +- [subworkflow_version](subworkflow_version) + - [`SubworkflowLint.subworkflow_version()`](subworkflow_version#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_version) diff --git a/src/content/tools/docs/2.10/subworkflow_lint_tests/main_nf.md b/src/content/tools/docs/2.10/subworkflow_lint_tests/main_nf.md new file mode 100644 index 0000000000..1280380868 --- /dev/null +++ b/src/content/tools/docs/2.10/subworkflow_lint_tests/main_nf.md @@ -0,0 +1,16 @@ +# main_nf + +#### `SubworkflowLint.main_nf(subworkflow){:python}` + +Lint a `main.nf` subworkflow file + +Can also be used to lint local subworkflow files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- A subworkflow SHOULD import at least two modules +- All included modules or subworkflows are used and their names are used for versions.yml +- The workflow name is all capital letters +- The subworkflow emits a software version diff --git a/src/content/tools/docs/2.10/subworkflow_lint_tests/meta_yml.md b/src/content/tools/docs/2.10/subworkflow_lint_tests/meta_yml.md new file mode 100644 index 0000000000..2ab290d54b --- /dev/null +++ b/src/content/tools/docs/2.10/subworkflow_lint_tests/meta_yml.md @@ -0,0 +1,17 @@ +# meta_yml + +#### `SubworkflowLint.meta_yml(subworkflow){:python}` + +Lint a `meta.yml` file + +The lint test checks that the subworkflow has +a `meta.yml` file and that it follows the +JSON schema defined in the `subworkflows/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the subworkflow name +and subworkflow input is consistent between the +`meta.yml` and the `main.nf`. + +Checks that all input and output channels are specified in `meta.yml`. +Checks that all included components in `main.nf` are specified in `meta.yml`. diff --git a/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_changes.md b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_changes.md new file mode 100644 index 0000000000..cda417f1b6 --- /dev/null +++ b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_changes.md @@ -0,0 +1,14 @@ +# subworkflow_changes + +#### `SubworkflowLint.subworkflow_changes(subworkflow){:python}` + +Checks whether installed nf-core subworkflow have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every subworkflow +and compares them to the local copies + +If the subworkflow has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_tests.md b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_tests.md new file mode 100644 index 0000000000..d4e9080415 --- /dev/null +++ b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_tests.md @@ -0,0 +1,12 @@ +# subworkflow_tests + +#### `SubworkflowLint.subworkflow_tests(subworkflow){:python}` + +Lint the tests of a subworkflow in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the subworkflow is present in the `pytest_modules.yml` +file. + +Additionally, hecks that all included components in test `main.nf` are specified in `test.yml` diff --git a/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_todos.md b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_todos.md new file mode 100644 index 0000000000..e17f92c01d --- /dev/null +++ b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_todos.md @@ -0,0 +1,28 @@ +# subworkflow_todos + +#### `SubworkflowLint.subworkflow_todos(subworkflow){:python}` + +Look for TODO statements in the subworkflow files + +The nf-core subworkflow template contains a number of comment lines to help developers +of new subworkflow know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the subworkflows and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_version.md b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_version.md new file mode 100644 index 0000000000..9b398909f7 --- /dev/null +++ b/src/content/tools/docs/2.10/subworkflow_lint_tests/subworkflow_version.md @@ -0,0 +1,9 @@ +# subworkflow_version + +#### `SubworkflowLint.subworkflow_version(subworkflow){:python}` + +Verifies that the subworkflow has a version specified in the `modules.json` file + +It checks whether the subworkflow has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the subworkflow available. diff --git a/src/content/tools/docs/2.11.1/api/bump_version.md b/src/content/tools/docs/2.11.1/api/bump_version.md new file mode 100644 index 0000000000..107f374a18 --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/bump_version.md @@ -0,0 +1,34 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename: str | Path, pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, patterns: List[Tuple[str, str]]){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.11.1/api/create.md b/src/content/tools/docs/2.11.1/api/create.md new file mode 100644 index 0000000000..756f822c77 --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/create.md @@ -0,0 +1,88 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain, pipeline_dir){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. + - **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + - **plain** (_bool_) – If true the pipeline template will be initialized plain, without customisation. + - **pipeline_dir** (_str_) – Path to the pipeline directory. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + **template_areas** (_list_) – List of available template areas to skip. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.11.1/api/download.md b/src/content/tools/docs/2.11.1/api/download.md new file mode 100644 index 0000000000..e2015e642b --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/download.md @@ -0,0 +1,309 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`exception{:python}`_`nf_core.download.ContainerError(container, registry, address, absolute_URI, out_path, singularity_command, error_msg){:python}` + +Bases: `Exception` + +A class of errors related to pulling containers with Singularity/Apptainer + +#### `undefined{:python}` ImageExists(error_log) + +Bases: `FileExistsError` + +Image already exists in cache/output directory. + +#### `undefined{:python}` ImageNotFound(error_log) + +Bases: `FileNotFoundError` + +The image can not be found in the registry + +#### `undefined{:python}` InvalidTag(error_log) + +Bases: `AttributeError` + +Image and registry are valid, but the (version) tag is not + +#### `undefined{:python}` OtherError(error_log) + +Bases: `RuntimeError` + +Undefined error with the container + +#### `undefined{:python}` RegistryNotFound(error_log) + +Bases: `ConnectionRefusedError` + +The specified registry does not resolve to a valid IP address + +### _`exception{:python}`_`nf_core.download.DownloadError{:python}` + +Bases: `RuntimeError` + +A custom exception that is raised when nf-core download encounters a problem that we already took into consideration. +In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, tower=False, download_configuration=None, container_system=None, container_library=None, container_cache_utilisation=None, container_cache_index=None, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_List_\*\[**str**]\*) – The workflow revision to download, like 1.0. Defaults to None. + - **container** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **tower** (_bool_) – Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(revision, wf_sha, download_url){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `download_workflow_static(){:python}` + +Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner. + +#### `download_workflow_tower(location=None){:python}` + +Create a bare-cloned git repository of the workflow, so it can be launched with tw launch as pipeline + +#### `find_container_images(workflow_directory){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic, +but not always, e.g. not for differentialabundance 1.2.0 + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw matches that will likely need to be cleaned. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(current_revision=''){:python}` + +Loop through container names and download Singularity images + +#### `prioritize_direct_download(container_list){:python}` + +Helper function that takes a list of container images (URLs and Docker URIs), +eliminates all Docker URIs for which also a URL is contained and returns the +cleaned and also deduplicated list. + +Conceptually, this works like so: + +Everything after the last Slash should be identical, e.g. “scanpy:1.7.2–pyhdfd78af_0” in +\[‘https://depot.galaxyproject.org/singularity/scanpy:1.7.2–pyhdfd78af\_0’, ‘biocontainers/scanpy:1.7.2–pyhdfd78af_0’] + +re.sub(‘.\*/(.\*)’,’1’,c) will drop everything up to the last slash from c (container_id) + +d.get(k:=re.sub(‘.\*/(.\*)’,’1’,c),’’) assigns the truncated string to k (key) and gets the +corresponding value from the dict if present or else defaults to “”. + +If the regex pattern matches, the original container_id will be assigned to the dict with the k key. +r”^$|(?!^http)” matches an empty string (we didn’t have it in the dict yet and want to keep it in either case) or +any string that does not start with http. Because if our current dict value already starts with http, +we want to keep it and not replace with with whatever we have now (which might be the Docker URI). + +A regex that matches http, r”^$|^http” could thus be used to prioritize the Docker URIs over http Downloads + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_config_inclusion(){:python}` + +Prompt for inclusion of institutional configurations + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch +Prompt user for revision tag if ‘–revision’ was not set +If –tower is specified, allow to select multiple revisions +Also the static download allows for multiple revisions, but +we do not prompt this option interactively. + +#### `prompt_singularity_cachedir_creation(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `prompt_singularity_cachedir_remote(){:python}` + +Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR + +#### `prompt_singularity_cachedir_utilization(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `read_remote_containers(){:python}` + +Reads the file specified as index for the remote Singularity cache dir + +#### `rectify_raw_container_matches(raw_findings){:python}` + +Helper function to rectify the raw extracted container matches into fully qualified container names. +If multiple containers are found, any prefixed with http for direct download is prioritized + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘biocontainers/fastqc:0.11.9–0’ }” + +Later DSL2, variable is being used: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity\_pull\_docker\_container ? +: “/${container_id}” : +“quay.io/biocontainers/${container_id}” }”
      +container_id = ‘mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0’ + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, library, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. + - **library** (_list_ _of_ _str_) – A list of libraries to try for pulling the image. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(revision_dirname){:python}` + +Edit the downloaded nextflow.config file to use the local config files + +### _`class{:python}`_`nf_core.download.WorkflowRepo(remote_url, revision, commit, location=None, hide_progress=False, in_cache=True){:python}` + +Bases: `SyncedRepo` + +An object to store details about a locally cached workflow repository. + +Important Attributes: +: fullname: The full name of the repository, `nf-core/{self.pipelinename}`. +local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to `$HOME/.cache/nf-core/nf-core/{self.pipeline}`. + +#### `access(){:python}` + +#### `bare_clone(destination){:python}` + +#### `checkout(commit){:python}` + +Checks out the repository at the requested commit + +- **Parameters:** + **commit** (_str_) – Git SHA of the commit + +#### `get_remote_branches(remote_url){:python}` + +Get all branches from a remote repository + +- **Parameters:** + **remote_url** (_str_) – The git url to the remote repository +- **Returns:** + All branches found in the remote +- **Return type:** + (set\[str]) + +#### `retry_setup_local_repo(skip_confirm=False){:python}` + +#### `setup_local_repo(remote, location=None, in_cache=True){:python}` + +Sets up the local git repository. If the repository has been cloned previously, it +returns a git.Repo object of that clone. Otherwise it tries to clone the repository from +the provided remote URL and returns a git.Repo of the new clone. + +- **Parameters:** + - **remote** (_str_) – git url of remote + - **location** (_Path_) – location where the clone should be created/cached. + - **in_cache** (_bool_\*,\* _optional_) – Whether to clone the repository from the cache. Defaults to False. + +Sets self.repo + +#### `tidy_tags_and_branches(){:python}` + +Function to delete all tags and branches that are not of interest to the downloader. +This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + +However, due to local caching, the downloader might also want access to revisions that had been deleted before. +In that case, don’t bother with re-adding the tags and rather download anew from Github. diff --git a/src/content/tools/docs/2.11.1/api/index.md b/src/content/tools/docs/2.11.1/api/index.md new file mode 100644 index 0000000000..fff6d0292e --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/index.md @@ -0,0 +1,78 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`ContainerError`](download#nf_core.download.ContainerError) + - [`DownloadError`](download#nf_core.download.DownloadError) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) + - [`WorkflowRepo`](download#nf_core.download.WorkflowRepo) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.params_file](params-file) + - [`ParamsFileBuilder`](params-file#nf_core.params_file.ParamsFileBuilder) + - [`_print_wrapped()`](params-file#nf_core.params_file._print_wrapped) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`SingularityCacheFilePathValidator`](utils#nf_core.utils.SingularityCacheFilePathValidator) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_remote_version()`](utils#nf_core.utils.fetch_remote_version) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nested_delitem()`](utils#nf_core.utils.nested_delitem) + - [`nested_setitem()`](utils#nf_core.utils.nested_setitem) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`run_cmd()`](utils#nf_core.utils.run_cmd) + - [`set_wd()`](utils#nf_core.utils.set_wd) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.11.1/api/launch.md b/src/content/tools/docs/2.11.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.11.1/api/licences.md b/src/content/tools/docs/2.11.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.11.1/api/lint.md b/src/content/tools/docs/2.11.1/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.11.1/api/list.md b/src/content/tools/docs/2.11.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.11.1/api/modules.md b/src/content/tools/docs/2.11.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.11.1/api/params-file.md b/src/content/tools/docs/2.11.1/api/params-file.md new file mode 100644 index 0000000000..6cef34b10a --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/params-file.md @@ -0,0 +1,84 @@ +# nf_core.params_file + +Create a YAML parameter file + +### _`class{:python}`_`nf_core.params_file.ParamsFileBuilder(pipeline=None, revision=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline. + +- **Parameters:** + - **pipeline** (_str_\*,\* _optional_) – Path to a local pipeline path or a remote pipeline. + - **revision** (_str_\*,\* _optional_) – Revision of the pipeline to use. + +#### `format_group(definition, show_hidden=False){:python}` + +Format a group of parameters of the schema as commented YAML. + +- **Parameters:** + - **definition** (_dict_) – Definition of the group from the schema + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for a group +- **Return type:** + str + +#### `format_param(name, properties, required_properties=(), show_hidden=False){:python}` + +Format a single parameter of the schema as commented YAML + +- **Parameters:** + - **name** (_str_) – Name of the parameter + - **properties** (_dict_) – Properties of the parameter + - **required_properties** (_list_) – List of required properties + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Section of a params-file.yml for given parameter + None: + > If the parameter is skipped because it is hidden and + > show_hidden is not set +- **Return type:** + str + +#### `generate_params_file(show_hidden=False){:python}` + +Generate the contents of a parameter template file. + +Assumes the pipeline has been fetched (if remote) and the schema loaded. + +- **Parameters:** + **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for the pipeline schema +- **Return type:** + str + +#### `get_pipeline(){:python}` + +Prompt the user for a pipeline name and get the schema + +#### `write_params_file(output_fn='nf-params.yaml', show_hidden=False, force=False){:python}` + +Build a template file for the pipeline schema. + +- **Parameters:** + - **output_fn** (_str_\*,\* _optional_) – Filename to write the template to. + - **show_hidden** (_bool_\*,\* _optional_) – Include parameters marked as hidden in the output + - **force** (_bool_\*,\* _optional_) – Whether to overwrite existing output file. +- **Returns:** + True if the template was written successfully, False otherwise +- **Return type:** + bool + +### `nf_core.params_file._print_wrapped(text, fill_char='-', mode='both', width=80, indent=0, drop_whitespace=True){:python}` + +Helper function to format text for the params-file template. + +- **Parameters:** + - **text** (_str_) – Text to print + - **fill_char** (_str_\*,\* _optional_) – Character to use for creating dividers. Defaults to ‘-‘. + - **mode** (_str_\*,\* _optional_) – Where to place dividers. Defaults to “both”. + - **width** (_int_\*,\* _optional_) – Maximum line-width of the output text. Defaults to 80. + - **indent** (_int_\*,\* _optional_) – Number of spaces to indent the text. Defaults to 0. + - **drop_whitespace** (_bool_\*,\* _optional_) – Whether to drop whitespace from the start and end of lines. diff --git a/src/content/tools/docs/2.11.1/api/schema.md b/src/content/tools/docs/2.11.1/api/schema.md new file mode 100644 index 0000000000..3ce5414c20 --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/schema.md @@ -0,0 +1,166 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema +Update defaults if they have changed + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.11.1/api/subworkflows.md b/src/content/tools/docs/2.11.1/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.11.1/api/sync.md b/src/content/tools/docs/2.11.1/api/sync.md new file mode 100644 index 0000000000..63da8cf89f --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/sync.md @@ -0,0 +1,152 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. DEPRECATED + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.11.1/api/utils.md b/src/content/tools/docs/2.11.1/api/utils.md new file mode 100644 index 0000000000..bb607ad688 --- /dev/null +++ b/src/content/tools/docs/2.11.1/api/utils.md @@ -0,0 +1,404 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### _`class{:python}`_`nf_core.utils.SingularityCacheFilePathValidator{:python}` + +Bases: `Validator` + +Validator for file path specified as –singularity-cache-index argument in nf-core download + +#### `_abc_impl{:python}`_= <\_abc.\_abc_data object>_ + +#### `validate(value){:python}` + +Validate the input. +If invalid, this should raise a `ValidationError`. + +- **Parameters:** + **document** – `Document` instance. + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_remote_version(source_url){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory: str | Path = '.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nested_delitem(d, keys){:python}` + +Deletes a key from a nested dictionary + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse, deleting the final one + +### `nf_core.utils.nested_setitem(d, keys, value){:python}` + +Sets the value in a nested dict using a list of keys to traverse + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse + - **value** (_Any_) – The value to be set for the last key in the chain + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API + - **multiple** (_bool_) – Allow selection of multiple releases & branches (for Tower) +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.run_cmd(executable: str, cmd: str){:python}` + +Run a specified command and capture the output. Handle errors nicely. + +### `nf_core.utils.set_wd(path: Path){:python}` + +Sets the working directory for this context. + +- **Parameters:** + **path** (_Path_) – Path to the working directory to be used inside this context. + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.11.1/index.md b/src/content/tools/docs/2.11.1/index.md new file mode 100644 index 0000000000..636ffaa787 --- /dev/null +++ b/src/content/tools/docs/2.11.1/index.md @@ -0,0 +1,12 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [Subworkflow code lint tests](subworkflow_lint_tests/index) (run by `nf-core subworkflows lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/index.md b/src/content/tools/docs/2.11.1/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/main_nf.md b/src/content/tools/docs/2.11.1/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..aa42cef6d7 --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, registry, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.11.1/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..30299022c5 --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/meta_yml.md @@ -0,0 +1,28 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module: NFCoreComponent){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/meta-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. + +If the module has inputs or outputs, they are expected to be +formatted as: + +..code-block:: +: tuple val(foo) path(bar) +val foo +path foo + +or permutations of the above. + +- **Parameters:** + - **module_lint_object** (_ComponentLint_) – The lint object for the module + - **module** (_NFCoreComponent_) – The module to lint diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/module_changes.md b/src/content/tools/docs/2.11.1/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.11.1/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/module_patch.md b/src/content/tools/docs/2.11.1/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..5d38a40d31 --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreComponent){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/module_tests.md b/src/content/tools/docs/2.11.1/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..df2872f29e --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/module_tests.md @@ -0,0 +1,8 @@ +# module_tests + +#### `ModuleLint.module_tests(module: NFCoreComponent){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/module_todos.md b/src/content/tools/docs/2.11.1/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.11.1/module_lint_tests/module_version.md b/src/content/tools/docs/2.11.1/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.11.1/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..aec43b98fd --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `NXF_VER` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + NXF_VER: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `NXF_VER` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..d65e9b5d8c --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +pipeline_template.yml # saving information in .nf-core.yml +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..6247c86c45 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,56 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..c8cf0fe7b1 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/index.md @@ -0,0 +1,44 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [system_exit](system_exit) + - [`PipelineLint.system_exit()`](system_exit#nf_core.lint.PipelineLint.system_exit) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..97dc2f769a --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,98 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.validationShowHiddenParams` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.validationSchemaIgnoreParams` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` + +**The configuration should contain the following or the test will fail:** + +- A `test` configuration profile should exist. diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.11.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.11.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.11.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.11.1/subworkflow_lint_tests/index.md b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/index.md new file mode 100644 index 0000000000..d5061028dc --- /dev/null +++ b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/index.md @@ -0,0 +1,16 @@ +# Subworkflow lint tests + +# Tests: + +- [main_nf](main_nf) + - [`SubworkflowLint.main_nf()`](main_nf#nf_core.subworkflows.lint.SubworkflowLint.main_nf) +- [meta_yml](meta_yml) + - [`SubworkflowLint.meta_yml()`](meta_yml#nf_core.subworkflows.lint.SubworkflowLint.meta_yml) +- [subworkflow_changes](subworkflow_changes) + - [`SubworkflowLint.subworkflow_changes()`](subworkflow_changes#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_changes) +- [subworkflow_tests](subworkflow_tests) + - [`SubworkflowLint.subworkflow_tests()`](subworkflow_tests#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_tests) +- [subworkflow_todos](subworkflow_todos) + - [`SubworkflowLint.subworkflow_todos()`](subworkflow_todos#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_todos) +- [subworkflow_version](subworkflow_version) + - [`SubworkflowLint.subworkflow_version()`](subworkflow_version#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_version) diff --git a/src/content/tools/docs/2.11.1/subworkflow_lint_tests/main_nf.md b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/main_nf.md new file mode 100644 index 0000000000..1280380868 --- /dev/null +++ b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/main_nf.md @@ -0,0 +1,16 @@ +# main_nf + +#### `SubworkflowLint.main_nf(subworkflow){:python}` + +Lint a `main.nf` subworkflow file + +Can also be used to lint local subworkflow files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- A subworkflow SHOULD import at least two modules +- All included modules or subworkflows are used and their names are used for versions.yml +- The workflow name is all capital letters +- The subworkflow emits a software version diff --git a/src/content/tools/docs/2.11.1/subworkflow_lint_tests/meta_yml.md b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/meta_yml.md new file mode 100644 index 0000000000..2ab290d54b --- /dev/null +++ b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/meta_yml.md @@ -0,0 +1,17 @@ +# meta_yml + +#### `SubworkflowLint.meta_yml(subworkflow){:python}` + +Lint a `meta.yml` file + +The lint test checks that the subworkflow has +a `meta.yml` file and that it follows the +JSON schema defined in the `subworkflows/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the subworkflow name +and subworkflow input is consistent between the +`meta.yml` and the `main.nf`. + +Checks that all input and output channels are specified in `meta.yml`. +Checks that all included components in `main.nf` are specified in `meta.yml`. diff --git a/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_changes.md b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_changes.md new file mode 100644 index 0000000000..cda417f1b6 --- /dev/null +++ b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_changes.md @@ -0,0 +1,14 @@ +# subworkflow_changes + +#### `SubworkflowLint.subworkflow_changes(subworkflow){:python}` + +Checks whether installed nf-core subworkflow have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every subworkflow +and compares them to the local copies + +If the subworkflow has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_tests.md b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_tests.md new file mode 100644 index 0000000000..240405567b --- /dev/null +++ b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_tests.md @@ -0,0 +1,10 @@ +# subworkflow_tests + +#### `SubworkflowLint.subworkflow_tests(subworkflow: NFCoreComponent){:python}` + +Lint the tests of a subworkflow in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. + +Additionally, hecks that all included components in test `main.nf` are specified in `test.yml` diff --git a/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_todos.md b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_todos.md new file mode 100644 index 0000000000..e17f92c01d --- /dev/null +++ b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_todos.md @@ -0,0 +1,28 @@ +# subworkflow_todos + +#### `SubworkflowLint.subworkflow_todos(subworkflow){:python}` + +Look for TODO statements in the subworkflow files + +The nf-core subworkflow template contains a number of comment lines to help developers +of new subworkflow know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the subworkflows and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_version.md b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_version.md new file mode 100644 index 0000000000..9b398909f7 --- /dev/null +++ b/src/content/tools/docs/2.11.1/subworkflow_lint_tests/subworkflow_version.md @@ -0,0 +1,9 @@ +# subworkflow_version + +#### `SubworkflowLint.subworkflow_version(subworkflow){:python}` + +Verifies that the subworkflow has a version specified in the `modules.json` file + +It checks whether the subworkflow has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the subworkflow available. diff --git a/src/content/tools/docs/2.11/api/bump_version.md b/src/content/tools/docs/2.11/api/bump_version.md new file mode 100644 index 0000000000..107f374a18 --- /dev/null +++ b/src/content/tools/docs/2.11/api/bump_version.md @@ -0,0 +1,34 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename: str | Path, pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, patterns: List[Tuple[str, str]]){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.11/api/create.md b/src/content/tools/docs/2.11/api/create.md new file mode 100644 index 0000000000..756f822c77 --- /dev/null +++ b/src/content/tools/docs/2.11/api/create.md @@ -0,0 +1,88 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain, pipeline_dir){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. + - **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + - **plain** (_bool_) – If true the pipeline template will be initialized plain, without customisation. + - **pipeline_dir** (_str_) – Path to the pipeline directory. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + **template_areas** (_list_) – List of available template areas to skip. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.11/api/download.md b/src/content/tools/docs/2.11/api/download.md new file mode 100644 index 0000000000..e2015e642b --- /dev/null +++ b/src/content/tools/docs/2.11/api/download.md @@ -0,0 +1,309 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`exception{:python}`_`nf_core.download.ContainerError(container, registry, address, absolute_URI, out_path, singularity_command, error_msg){:python}` + +Bases: `Exception` + +A class of errors related to pulling containers with Singularity/Apptainer + +#### `undefined{:python}` ImageExists(error_log) + +Bases: `FileExistsError` + +Image already exists in cache/output directory. + +#### `undefined{:python}` ImageNotFound(error_log) + +Bases: `FileNotFoundError` + +The image can not be found in the registry + +#### `undefined{:python}` InvalidTag(error_log) + +Bases: `AttributeError` + +Image and registry are valid, but the (version) tag is not + +#### `undefined{:python}` OtherError(error_log) + +Bases: `RuntimeError` + +Undefined error with the container + +#### `undefined{:python}` RegistryNotFound(error_log) + +Bases: `ConnectionRefusedError` + +The specified registry does not resolve to a valid IP address + +### _`exception{:python}`_`nf_core.download.DownloadError{:python}` + +Bases: `RuntimeError` + +A custom exception that is raised when nf-core download encounters a problem that we already took into consideration. +In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, tower=False, download_configuration=None, container_system=None, container_library=None, container_cache_utilisation=None, container_cache_index=None, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_List_\*\[**str**]\*) – The workflow revision to download, like 1.0. Defaults to None. + - **container** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **tower** (_bool_) – Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(revision, wf_sha, download_url){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `download_workflow_static(){:python}` + +Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner. + +#### `download_workflow_tower(location=None){:python}` + +Create a bare-cloned git repository of the workflow, so it can be launched with tw launch as pipeline + +#### `find_container_images(workflow_directory){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic, +but not always, e.g. not for differentialabundance 1.2.0 + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw matches that will likely need to be cleaned. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(current_revision=''){:python}` + +Loop through container names and download Singularity images + +#### `prioritize_direct_download(container_list){:python}` + +Helper function that takes a list of container images (URLs and Docker URIs), +eliminates all Docker URIs for which also a URL is contained and returns the +cleaned and also deduplicated list. + +Conceptually, this works like so: + +Everything after the last Slash should be identical, e.g. “scanpy:1.7.2–pyhdfd78af_0” in +\[‘https://depot.galaxyproject.org/singularity/scanpy:1.7.2–pyhdfd78af\_0’, ‘biocontainers/scanpy:1.7.2–pyhdfd78af_0’] + +re.sub(‘.\*/(.\*)’,’1’,c) will drop everything up to the last slash from c (container_id) + +d.get(k:=re.sub(‘.\*/(.\*)’,’1’,c),’’) assigns the truncated string to k (key) and gets the +corresponding value from the dict if present or else defaults to “”. + +If the regex pattern matches, the original container_id will be assigned to the dict with the k key. +r”^$|(?!^http)” matches an empty string (we didn’t have it in the dict yet and want to keep it in either case) or +any string that does not start with http. Because if our current dict value already starts with http, +we want to keep it and not replace with with whatever we have now (which might be the Docker URI). + +A regex that matches http, r”^$|^http” could thus be used to prioritize the Docker URIs over http Downloads + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_config_inclusion(){:python}` + +Prompt for inclusion of institutional configurations + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch +Prompt user for revision tag if ‘–revision’ was not set +If –tower is specified, allow to select multiple revisions +Also the static download allows for multiple revisions, but +we do not prompt this option interactively. + +#### `prompt_singularity_cachedir_creation(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `prompt_singularity_cachedir_remote(){:python}` + +Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR + +#### `prompt_singularity_cachedir_utilization(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `read_remote_containers(){:python}` + +Reads the file specified as index for the remote Singularity cache dir + +#### `rectify_raw_container_matches(raw_findings){:python}` + +Helper function to rectify the raw extracted container matches into fully qualified container names. +If multiple containers are found, any prefixed with http for direct download is prioritized + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘biocontainers/fastqc:0.11.9–0’ }” + +Later DSL2, variable is being used: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity\_pull\_docker\_container ? +: “/${container_id}” : +“quay.io/biocontainers/${container_id}” }”
      +container_id = ‘mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0’ + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, library, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. + - **library** (_list_ _of_ _str_) – A list of libraries to try for pulling the image. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(revision_dirname){:python}` + +Edit the downloaded nextflow.config file to use the local config files + +### _`class{:python}`_`nf_core.download.WorkflowRepo(remote_url, revision, commit, location=None, hide_progress=False, in_cache=True){:python}` + +Bases: `SyncedRepo` + +An object to store details about a locally cached workflow repository. + +Important Attributes: +: fullname: The full name of the repository, `nf-core/{self.pipelinename}`. +local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to `$HOME/.cache/nf-core/nf-core/{self.pipeline}`. + +#### `access(){:python}` + +#### `bare_clone(destination){:python}` + +#### `checkout(commit){:python}` + +Checks out the repository at the requested commit + +- **Parameters:** + **commit** (_str_) – Git SHA of the commit + +#### `get_remote_branches(remote_url){:python}` + +Get all branches from a remote repository + +- **Parameters:** + **remote_url** (_str_) – The git url to the remote repository +- **Returns:** + All branches found in the remote +- **Return type:** + (set\[str]) + +#### `retry_setup_local_repo(skip_confirm=False){:python}` + +#### `setup_local_repo(remote, location=None, in_cache=True){:python}` + +Sets up the local git repository. If the repository has been cloned previously, it +returns a git.Repo object of that clone. Otherwise it tries to clone the repository from +the provided remote URL and returns a git.Repo of the new clone. + +- **Parameters:** + - **remote** (_str_) – git url of remote + - **location** (_Path_) – location where the clone should be created/cached. + - **in_cache** (_bool_\*,\* _optional_) – Whether to clone the repository from the cache. Defaults to False. + +Sets self.repo + +#### `tidy_tags_and_branches(){:python}` + +Function to delete all tags and branches that are not of interest to the downloader. +This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + +However, due to local caching, the downloader might also want access to revisions that had been deleted before. +In that case, don’t bother with re-adding the tags and rather download anew from Github. diff --git a/src/content/tools/docs/2.11/api/index.md b/src/content/tools/docs/2.11/api/index.md new file mode 100644 index 0000000000..fff6d0292e --- /dev/null +++ b/src/content/tools/docs/2.11/api/index.md @@ -0,0 +1,78 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`ContainerError`](download#nf_core.download.ContainerError) + - [`DownloadError`](download#nf_core.download.DownloadError) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) + - [`WorkflowRepo`](download#nf_core.download.WorkflowRepo) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.params_file](params-file) + - [`ParamsFileBuilder`](params-file#nf_core.params_file.ParamsFileBuilder) + - [`_print_wrapped()`](params-file#nf_core.params_file._print_wrapped) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`SingularityCacheFilePathValidator`](utils#nf_core.utils.SingularityCacheFilePathValidator) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_remote_version()`](utils#nf_core.utils.fetch_remote_version) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nested_delitem()`](utils#nf_core.utils.nested_delitem) + - [`nested_setitem()`](utils#nf_core.utils.nested_setitem) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`run_cmd()`](utils#nf_core.utils.run_cmd) + - [`set_wd()`](utils#nf_core.utils.set_wd) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.11/api/launch.md b/src/content/tools/docs/2.11/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.11/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.11/api/licences.md b/src/content/tools/docs/2.11/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.11/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.11/api/lint.md b/src/content/tools/docs/2.11/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.11/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.11/api/list.md b/src/content/tools/docs/2.11/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.11/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.11/api/modules.md b/src/content/tools/docs/2.11/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.11/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.11/api/params-file.md b/src/content/tools/docs/2.11/api/params-file.md new file mode 100644 index 0000000000..6cef34b10a --- /dev/null +++ b/src/content/tools/docs/2.11/api/params-file.md @@ -0,0 +1,84 @@ +# nf_core.params_file + +Create a YAML parameter file + +### _`class{:python}`_`nf_core.params_file.ParamsFileBuilder(pipeline=None, revision=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline. + +- **Parameters:** + - **pipeline** (_str_\*,\* _optional_) – Path to a local pipeline path or a remote pipeline. + - **revision** (_str_\*,\* _optional_) – Revision of the pipeline to use. + +#### `format_group(definition, show_hidden=False){:python}` + +Format a group of parameters of the schema as commented YAML. + +- **Parameters:** + - **definition** (_dict_) – Definition of the group from the schema + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for a group +- **Return type:** + str + +#### `format_param(name, properties, required_properties=(), show_hidden=False){:python}` + +Format a single parameter of the schema as commented YAML + +- **Parameters:** + - **name** (_str_) – Name of the parameter + - **properties** (_dict_) – Properties of the parameter + - **required_properties** (_list_) – List of required properties + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Section of a params-file.yml for given parameter + None: + > If the parameter is skipped because it is hidden and + > show_hidden is not set +- **Return type:** + str + +#### `generate_params_file(show_hidden=False){:python}` + +Generate the contents of a parameter template file. + +Assumes the pipeline has been fetched (if remote) and the schema loaded. + +- **Parameters:** + **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for the pipeline schema +- **Return type:** + str + +#### `get_pipeline(){:python}` + +Prompt the user for a pipeline name and get the schema + +#### `write_params_file(output_fn='nf-params.yaml', show_hidden=False, force=False){:python}` + +Build a template file for the pipeline schema. + +- **Parameters:** + - **output_fn** (_str_\*,\* _optional_) – Filename to write the template to. + - **show_hidden** (_bool_\*,\* _optional_) – Include parameters marked as hidden in the output + - **force** (_bool_\*,\* _optional_) – Whether to overwrite existing output file. +- **Returns:** + True if the template was written successfully, False otherwise +- **Return type:** + bool + +### `nf_core.params_file._print_wrapped(text, fill_char='-', mode='both', width=80, indent=0, drop_whitespace=True){:python}` + +Helper function to format text for the params-file template. + +- **Parameters:** + - **text** (_str_) – Text to print + - **fill_char** (_str_\*,\* _optional_) – Character to use for creating dividers. Defaults to ‘-‘. + - **mode** (_str_\*,\* _optional_) – Where to place dividers. Defaults to “both”. + - **width** (_int_\*,\* _optional_) – Maximum line-width of the output text. Defaults to 80. + - **indent** (_int_\*,\* _optional_) – Number of spaces to indent the text. Defaults to 0. + - **drop_whitespace** (_bool_\*,\* _optional_) – Whether to drop whitespace from the start and end of lines. diff --git a/src/content/tools/docs/2.11/api/schema.md b/src/content/tools/docs/2.11/api/schema.md new file mode 100644 index 0000000000..3ce5414c20 --- /dev/null +++ b/src/content/tools/docs/2.11/api/schema.md @@ -0,0 +1,166 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema +Update defaults if they have changed + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.11/api/subworkflows.md b/src/content/tools/docs/2.11/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.11/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.11/api/sync.md b/src/content/tools/docs/2.11/api/sync.md new file mode 100644 index 0000000000..63da8cf89f --- /dev/null +++ b/src/content/tools/docs/2.11/api/sync.md @@ -0,0 +1,152 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. DEPRECATED + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.11/api/utils.md b/src/content/tools/docs/2.11/api/utils.md new file mode 100644 index 0000000000..bb607ad688 --- /dev/null +++ b/src/content/tools/docs/2.11/api/utils.md @@ -0,0 +1,404 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### _`class{:python}`_`nf_core.utils.SingularityCacheFilePathValidator{:python}` + +Bases: `Validator` + +Validator for file path specified as –singularity-cache-index argument in nf-core download + +#### `_abc_impl{:python}`_= <\_abc.\_abc_data object>_ + +#### `validate(value){:python}` + +Validate the input. +If invalid, this should raise a `ValidationError`. + +- **Parameters:** + **document** – `Document` instance. + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_remote_version(source_url){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory: str | Path = '.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nested_delitem(d, keys){:python}` + +Deletes a key from a nested dictionary + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse, deleting the final one + +### `nf_core.utils.nested_setitem(d, keys, value){:python}` + +Sets the value in a nested dict using a list of keys to traverse + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse + - **value** (_Any_) – The value to be set for the last key in the chain + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API + - **multiple** (_bool_) – Allow selection of multiple releases & branches (for Tower) +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.run_cmd(executable: str, cmd: str){:python}` + +Run a specified command and capture the output. Handle errors nicely. + +### `nf_core.utils.set_wd(path: Path){:python}` + +Sets the working directory for this context. + +- **Parameters:** + **path** (_Path_) – Path to the working directory to be used inside this context. + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.11/index.md b/src/content/tools/docs/2.11/index.md new file mode 100644 index 0000000000..636ffaa787 --- /dev/null +++ b/src/content/tools/docs/2.11/index.md @@ -0,0 +1,12 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [Subworkflow code lint tests](subworkflow_lint_tests/index) (run by `nf-core subworkflows lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.11/module_lint_tests/index.md b/src/content/tools/docs/2.11/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.11/module_lint_tests/main_nf.md b/src/content/tools/docs/2.11/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..aa42cef6d7 --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, registry, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.11/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.11/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..30299022c5 --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/meta_yml.md @@ -0,0 +1,28 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module: NFCoreComponent){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/meta-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. + +If the module has inputs or outputs, they are expected to be +formatted as: + +..code-block:: +: tuple val(foo) path(bar) +val foo +path foo + +or permutations of the above. + +- **Parameters:** + - **module_lint_object** (_ComponentLint_) – The lint object for the module + - **module** (_NFCoreComponent_) – The module to lint diff --git a/src/content/tools/docs/2.11/module_lint_tests/module_changes.md b/src/content/tools/docs/2.11/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.11/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.11/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.11/module_lint_tests/module_patch.md b/src/content/tools/docs/2.11/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..5d38a40d31 --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreComponent){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.11/module_lint_tests/module_tests.md b/src/content/tools/docs/2.11/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..df2872f29e --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/module_tests.md @@ -0,0 +1,8 @@ +# module_tests + +#### `ModuleLint.module_tests(module: NFCoreComponent){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. diff --git a/src/content/tools/docs/2.11/module_lint_tests/module_todos.md b/src/content/tools/docs/2.11/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.11/module_lint_tests/module_version.md b/src/content/tools/docs/2.11/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.11/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..aec43b98fd --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `NXF_VER` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + NXF_VER: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `NXF_VER` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.11/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..d65e9b5d8c --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +pipeline_template.yml # saving information in .nf-core.yml +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.11/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..6247c86c45 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,56 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/index.md b/src/content/tools/docs/2.11/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..c8cf0fe7b1 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/index.md @@ -0,0 +1,44 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [system_exit](system_exit) + - [`PipelineLint.system_exit()`](system_exit#nf_core.lint.PipelineLint.system_exit) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.11/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.11/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.11/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.11/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.11/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..97dc2f769a --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,98 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.validationShowHiddenParams` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.validationSchemaIgnoreParams` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` + +**The configuration should contain the following or the test will fail:** + +- A `test` configuration profile should exist. diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.11/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.11/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.11/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.11/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.11/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.11/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/2.11/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.11/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.11/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.11/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.11/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.11/subworkflow_lint_tests/index.md b/src/content/tools/docs/2.11/subworkflow_lint_tests/index.md new file mode 100644 index 0000000000..d5061028dc --- /dev/null +++ b/src/content/tools/docs/2.11/subworkflow_lint_tests/index.md @@ -0,0 +1,16 @@ +# Subworkflow lint tests + +# Tests: + +- [main_nf](main_nf) + - [`SubworkflowLint.main_nf()`](main_nf#nf_core.subworkflows.lint.SubworkflowLint.main_nf) +- [meta_yml](meta_yml) + - [`SubworkflowLint.meta_yml()`](meta_yml#nf_core.subworkflows.lint.SubworkflowLint.meta_yml) +- [subworkflow_changes](subworkflow_changes) + - [`SubworkflowLint.subworkflow_changes()`](subworkflow_changes#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_changes) +- [subworkflow_tests](subworkflow_tests) + - [`SubworkflowLint.subworkflow_tests()`](subworkflow_tests#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_tests) +- [subworkflow_todos](subworkflow_todos) + - [`SubworkflowLint.subworkflow_todos()`](subworkflow_todos#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_todos) +- [subworkflow_version](subworkflow_version) + - [`SubworkflowLint.subworkflow_version()`](subworkflow_version#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_version) diff --git a/src/content/tools/docs/2.11/subworkflow_lint_tests/main_nf.md b/src/content/tools/docs/2.11/subworkflow_lint_tests/main_nf.md new file mode 100644 index 0000000000..1280380868 --- /dev/null +++ b/src/content/tools/docs/2.11/subworkflow_lint_tests/main_nf.md @@ -0,0 +1,16 @@ +# main_nf + +#### `SubworkflowLint.main_nf(subworkflow){:python}` + +Lint a `main.nf` subworkflow file + +Can also be used to lint local subworkflow files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- A subworkflow SHOULD import at least two modules +- All included modules or subworkflows are used and their names are used for versions.yml +- The workflow name is all capital letters +- The subworkflow emits a software version diff --git a/src/content/tools/docs/2.11/subworkflow_lint_tests/meta_yml.md b/src/content/tools/docs/2.11/subworkflow_lint_tests/meta_yml.md new file mode 100644 index 0000000000..2ab290d54b --- /dev/null +++ b/src/content/tools/docs/2.11/subworkflow_lint_tests/meta_yml.md @@ -0,0 +1,17 @@ +# meta_yml + +#### `SubworkflowLint.meta_yml(subworkflow){:python}` + +Lint a `meta.yml` file + +The lint test checks that the subworkflow has +a `meta.yml` file and that it follows the +JSON schema defined in the `subworkflows/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the subworkflow name +and subworkflow input is consistent between the +`meta.yml` and the `main.nf`. + +Checks that all input and output channels are specified in `meta.yml`. +Checks that all included components in `main.nf` are specified in `meta.yml`. diff --git a/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_changes.md b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_changes.md new file mode 100644 index 0000000000..cda417f1b6 --- /dev/null +++ b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_changes.md @@ -0,0 +1,14 @@ +# subworkflow_changes + +#### `SubworkflowLint.subworkflow_changes(subworkflow){:python}` + +Checks whether installed nf-core subworkflow have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every subworkflow +and compares them to the local copies + +If the subworkflow has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_tests.md b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_tests.md new file mode 100644 index 0000000000..240405567b --- /dev/null +++ b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_tests.md @@ -0,0 +1,10 @@ +# subworkflow_tests + +#### `SubworkflowLint.subworkflow_tests(subworkflow: NFCoreComponent){:python}` + +Lint the tests of a subworkflow in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. + +Additionally, hecks that all included components in test `main.nf` are specified in `test.yml` diff --git a/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_todos.md b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_todos.md new file mode 100644 index 0000000000..e17f92c01d --- /dev/null +++ b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_todos.md @@ -0,0 +1,28 @@ +# subworkflow_todos + +#### `SubworkflowLint.subworkflow_todos(subworkflow){:python}` + +Look for TODO statements in the subworkflow files + +The nf-core subworkflow template contains a number of comment lines to help developers +of new subworkflow know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the subworkflows and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_version.md b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_version.md new file mode 100644 index 0000000000..9b398909f7 --- /dev/null +++ b/src/content/tools/docs/2.11/subworkflow_lint_tests/subworkflow_version.md @@ -0,0 +1,9 @@ +# subworkflow_version + +#### `SubworkflowLint.subworkflow_version(subworkflow){:python}` + +Verifies that the subworkflow has a version specified in the `modules.json` file + +It checks whether the subworkflow has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the subworkflow available. diff --git a/src/content/tools/docs/2.12.1/api/bump_version.md b/src/content/tools/docs/2.12.1/api/bump_version.md new file mode 100644 index 0000000000..107f374a18 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/bump_version.md @@ -0,0 +1,34 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename: str | Path, pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, patterns: List[Tuple[str, str]]){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.12.1/api/create.md b/src/content/tools/docs/2.12.1/api/create.md new file mode 100644 index 0000000000..048529adb0 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain, pipeline_dir){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. + - **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + - **plain** (_bool_) – If true the pipeline template will be initialized plain, without customisation. + - **pipeline_dir** (_str_) – Path to the pipeline directory. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + **template_areas** (_list_) – List of available template areas to skip. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.12.1/api/download.md b/src/content/tools/docs/2.12.1/api/download.md new file mode 100644 index 0000000000..1a99ec6506 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/download.md @@ -0,0 +1,309 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`exception{:python}`_`nf_core.download.ContainerError(container, registry, address, absolute_URI, out_path, singularity_command, error_msg){:python}` + +Bases: `Exception` + +A class of errors related to pulling containers with Singularity/Apptainer + +#### `undefined{:python}` ImageExistsError(error_log) + +Bases: `FileExistsError` + +Image already exists in cache/output directory. + +#### `undefined{:python}` ImageNotFoundError(error_log) + +Bases: `FileNotFoundError` + +The image can not be found in the registry + +#### `undefined{:python}` InvalidTagError(error_log) + +Bases: `AttributeError` + +Image and registry are valid, but the (version) tag is not + +#### `undefined{:python}` OtherError(error_log) + +Bases: `RuntimeError` + +Undefined error with the container + +#### `undefined{:python}` RegistryNotFoundError(error_log) + +Bases: `ConnectionRefusedError` + +The specified registry does not resolve to a valid IP address + +### _`exception{:python}`_`nf_core.download.DownloadError{:python}` + +Bases: `RuntimeError` + +A custom exception that is raised when nf-core download encounters a problem that we already took into consideration. +In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, tower=False, download_configuration=None, container_system=None, container_library=None, container_cache_utilisation=None, container_cache_index=None, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_List_\*\[**str**]\*) – The workflow revision to download, like 1.0. Defaults to None. + - **container** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **tower** (_bool_) – Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(revision, wf_sha, download_url){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `download_workflow_static(){:python}` + +Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner. + +#### `download_workflow_tower(location=None){:python}` + +Create a bare-cloned git repository of the workflow, so it can be launched with tw launch as pipeline + +#### `find_container_images(workflow_directory){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic, +but not always, e.g. not for differentialabundance 1.2.0 + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw matches that will likely need to be cleaned. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(current_revision=''){:python}` + +Loop through container names and download Singularity images + +#### `prioritize_direct_download(container_list){:python}` + +Helper function that takes a list of container images (URLs and Docker URIs), +eliminates all Docker URIs for which also a URL is contained and returns the +cleaned and also deduplicated list. + +Conceptually, this works like so: + +Everything after the last Slash should be identical, e.g. “scanpy:1.7.2–pyhdfd78af_0” in +\[‘https://depot.galaxyproject.org/singularity/scanpy:1.7.2–pyhdfd78af\_0’, ‘biocontainers/scanpy:1.7.2–pyhdfd78af_0’] + +re.sub(‘.\*/(.\*)’,’1’,c) will drop everything up to the last slash from c (container_id) + +d.get(k:=re.sub(‘.\*/(.\*)’,’1’,c),’’) assigns the truncated string to k (key) and gets the +corresponding value from the dict if present or else defaults to “”. + +If the regex pattern matches, the original container_id will be assigned to the dict with the k key. +r”^$|(?!^http)” matches an empty string (we didn’t have it in the dict yet and want to keep it in either case) or +any string that does not start with http. Because if our current dict value already starts with http, +we want to keep it and not replace with with whatever we have now (which might be the Docker URI). + +A regex that matches http, r”^$|^http” could thus be used to prioritize the Docker URIs over http Downloads + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_config_inclusion(){:python}` + +Prompt for inclusion of institutional configurations + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch +Prompt user for revision tag if ‘–revision’ was not set +If –tower is specified, allow to select multiple revisions +Also the static download allows for multiple revisions, but +we do not prompt this option interactively. + +#### `prompt_singularity_cachedir_creation(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `prompt_singularity_cachedir_remote(){:python}` + +Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR + +#### `prompt_singularity_cachedir_utilization(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `read_remote_containers(){:python}` + +Reads the file specified as index for the remote Singularity cache dir + +#### `rectify_raw_container_matches(raw_findings){:python}` + +Helper function to rectify the raw extracted container matches into fully qualified container names. +If multiple containers are found, any prefixed with http for direct download is prioritized + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘biocontainers/fastqc:0.11.9–0’ }” + +Later DSL2, variable is being used: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity\_pull\_docker\_container ? +: “/${container_id}” : +“quay.io/biocontainers/${container_id}” }”
      +container_id = ‘mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0’ + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, library, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. + - **library** (_list_ _of_ _str_) – A list of libraries to try for pulling the image. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(revision_dirname){:python}` + +Edit the downloaded nextflow.config file to use the local config files + +### _`class{:python}`_`nf_core.download.WorkflowRepo(remote_url, revision, commit, location=None, hide_progress=False, in_cache=True){:python}` + +Bases: `SyncedRepo` + +An object to store details about a locally cached workflow repository. + +Important Attributes: +: fullname: The full name of the repository, `nf-core/{self.pipelinename}`. +local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to `$HOME/.cache/nf-core/nf-core/{self.pipeline}`. + +#### `access(){:python}` + +#### `bare_clone(destination){:python}` + +#### `checkout(commit){:python}` + +Checks out the repository at the requested commit + +- **Parameters:** + **commit** (_str_) – Git SHA of the commit + +#### `get_remote_branches(remote_url){:python}` + +Get all branches from a remote repository + +- **Parameters:** + **remote_url** (_str_) – The git url to the remote repository +- **Returns:** + All branches found in the remote +- **Return type:** + (set\[str]) + +#### `retry_setup_local_repo(skip_confirm=False){:python}` + +#### `setup_local_repo(remote, location=None, in_cache=True){:python}` + +Sets up the local git repository. If the repository has been cloned previously, it +returns a git.Repo object of that clone. Otherwise it tries to clone the repository from +the provided remote URL and returns a git.Repo of the new clone. + +- **Parameters:** + - **remote** (_str_) – git url of remote + - **location** (_Path_) – location where the clone should be created/cached. + - **in_cache** (_bool_\*,\* _optional_) – Whether to clone the repository from the cache. Defaults to False. + +Sets self.repo + +#### `tidy_tags_and_branches(){:python}` + +Function to delete all tags and branches that are not of interest to the downloader. +This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + +However, due to local caching, the downloader might also want access to revisions that had been deleted before. +In that case, don’t bother with re-adding the tags and rather download anew from Github. diff --git a/src/content/tools/docs/2.12.1/api/index.md b/src/content/tools/docs/2.12.1/api/index.md new file mode 100644 index 0000000000..340d966970 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/index.md @@ -0,0 +1,79 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`ContainerError`](download#nf_core.download.ContainerError) + - [`DownloadError`](download#nf_core.download.DownloadError) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) + - [`WorkflowRepo`](download#nf_core.download.WorkflowRepo) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.params_file](params-file) + - [`ParamsFileBuilder`](params-file#nf_core.params_file.ParamsFileBuilder) + - [`_print_wrapped()`](params-file#nf_core.params_file._print_wrapped) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestExceptionError`](sync#nf_core.sync.PullRequestExceptionError) + - [`SyncExceptionError`](sync#nf_core.sync.SyncExceptionError) +- [nf_core.utils](utils) + - [`GitHubAPISession`](utils#nf_core.utils.GitHubAPISession) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`SingularityCacheFilePathValidator`](utils#nf_core.utils.SingularityCacheFilePathValidator) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_remote_version()`](utils#nf_core.utils.fetch_remote_version) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nested_delitem()`](utils#nf_core.utils.nested_delitem) + - [`nested_setitem()`](utils#nf_core.utils.nested_setitem) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`run_cmd()`](utils#nf_core.utils.run_cmd) + - [`set_wd()`](utils#nf_core.utils.set_wd) + - [`setup_nfcore_cachedir()`](utils#nf_core.utils.setup_nfcore_cachedir) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.12.1/api/launch.md b/src/content/tools/docs/2.12.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.12.1/api/licences.md b/src/content/tools/docs/2.12.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.12.1/api/lint.md b/src/content/tools/docs/2.12.1/api/lint.md new file mode 100644 index 0000000000..abc826b708 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/lint.md @@ -0,0 +1,134 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + An object of type `ComponentLint` that contains all the linting results for the modules. + An object of type `ComponentLint` that contains all the linting results for the subworkflows. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files: List[str] | List[Path] | Path){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.12.1/api/list.md b/src/content/tools/docs/2.12.1/api/list.md new file mode 100644 index 0000000000..8e58cd9854 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow: str | Path, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.12.1/api/modules.md b/src/content/tools/docs/2.12.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.12.1/api/params-file.md b/src/content/tools/docs/2.12.1/api/params-file.md new file mode 100644 index 0000000000..6cef34b10a --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/params-file.md @@ -0,0 +1,84 @@ +# nf_core.params_file + +Create a YAML parameter file + +### _`class{:python}`_`nf_core.params_file.ParamsFileBuilder(pipeline=None, revision=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline. + +- **Parameters:** + - **pipeline** (_str_\*,\* _optional_) – Path to a local pipeline path or a remote pipeline. + - **revision** (_str_\*,\* _optional_) – Revision of the pipeline to use. + +#### `format_group(definition, show_hidden=False){:python}` + +Format a group of parameters of the schema as commented YAML. + +- **Parameters:** + - **definition** (_dict_) – Definition of the group from the schema + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for a group +- **Return type:** + str + +#### `format_param(name, properties, required_properties=(), show_hidden=False){:python}` + +Format a single parameter of the schema as commented YAML + +- **Parameters:** + - **name** (_str_) – Name of the parameter + - **properties** (_dict_) – Properties of the parameter + - **required_properties** (_list_) – List of required properties + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Section of a params-file.yml for given parameter + None: + > If the parameter is skipped because it is hidden and + > show_hidden is not set +- **Return type:** + str + +#### `generate_params_file(show_hidden=False){:python}` + +Generate the contents of a parameter template file. + +Assumes the pipeline has been fetched (if remote) and the schema loaded. + +- **Parameters:** + **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for the pipeline schema +- **Return type:** + str + +#### `get_pipeline(){:python}` + +Prompt the user for a pipeline name and get the schema + +#### `write_params_file(output_fn='nf-params.yaml', show_hidden=False, force=False){:python}` + +Build a template file for the pipeline schema. + +- **Parameters:** + - **output_fn** (_str_\*,\* _optional_) – Filename to write the template to. + - **show_hidden** (_bool_\*,\* _optional_) – Include parameters marked as hidden in the output + - **force** (_bool_\*,\* _optional_) – Whether to overwrite existing output file. +- **Returns:** + True if the template was written successfully, False otherwise +- **Return type:** + bool + +### `nf_core.params_file._print_wrapped(text, fill_char='-', mode='both', width=80, indent=0, drop_whitespace=True){:python}` + +Helper function to format text for the params-file template. + +- **Parameters:** + - **text** (_str_) – Text to print + - **fill_char** (_str_\*,\* _optional_) – Character to use for creating dividers. Defaults to ‘-‘. + - **mode** (_str_\*,\* _optional_) – Where to place dividers. Defaults to “both”. + - **width** (_int_\*,\* _optional_) – Maximum line-width of the output text. Defaults to 80. + - **indent** (_int_\*,\* _optional_) – Number of spaces to indent the text. Defaults to 0. + - **drop_whitespace** (_bool_\*,\* _optional_) – Whether to drop whitespace from the start and end of lines. diff --git a/src/content/tools/docs/2.12.1/api/schema.md b/src/content/tools/docs/2.12.1/api/schema.md new file mode 100644 index 0000000000..d6cced3692 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/schema.md @@ -0,0 +1,170 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema +Update defaults if they have changed + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_schema_types(){:python}` + +Get a list of all parameter types in the schema + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.12.1/api/subworkflows.md b/src/content/tools/docs/2.12.1/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.12.1/api/sync.md b/src/content/tools/docs/2.12.1/api/sync.md new file mode 100644 index 0000000000..bc7034cf1c --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/sync.md @@ -0,0 +1,152 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. DEPRECATED + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestExceptionError{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncExceptionError{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.12.1/api/utils.md b/src/content/tools/docs/2.12.1/api/utils.md new file mode 100644 index 0000000000..4b4cae4ea6 --- /dev/null +++ b/src/content/tools/docs/2.12.1/api/utils.md @@ -0,0 +1,408 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHubAPISession{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### _`class{:python}`_`nf_core.utils.SingularityCacheFilePathValidator{:python}` + +Bases: `Validator` + +Validator for file path specified as –singularity-cache-index argument in nf-core download + +#### `_abc_impl{:python}`_= <\_abc.\_abc_data object>_ + +#### `validate(value){:python}` + +Validate the input. +If invalid, this should raise a `ValidationError`. + +- **Parameters:** + **document** – `Document` instance. + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_remote_version(source_url){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory: str | Path = '.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nested_delitem(d, keys){:python}` + +Deletes a key from a nested dictionary + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse, deleting the final one + +### `nf_core.utils.nested_setitem(d, keys, value){:python}` + +Sets the value in a nested dict using a list of keys to traverse + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse + - **value** (_Any_) – The value to be set for the last key in the chain + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API + - **multiple** (_bool_) – Allow selection of multiple releases & branches (for Tower) +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.run_cmd(executable: str, cmd: str){:python}` + +Run a specified command and capture the output. Handle errors nicely. + +### `nf_core.utils.set_wd(path: Path){:python}` + +Sets the working directory for this context. + +- **Parameters:** + **path** (_Path_) – Path to the working directory to be used inside this context. + +### `nf_core.utils.setup_nfcore_cachedir(cache_fn: str | Path){:python}` + +Sets up local caching for caching files between sessions. + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.12.1/index.md b/src/content/tools/docs/2.12.1/index.md new file mode 100644 index 0000000000..636ffaa787 --- /dev/null +++ b/src/content/tools/docs/2.12.1/index.md @@ -0,0 +1,12 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [Subworkflow code lint tests](subworkflow_lint_tests/index) (run by `nf-core subworkflows lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/index.md b/src/content/tools/docs/2.12.1/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/main_nf.md b/src/content/tools/docs/2.12.1/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..aa42cef6d7 --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, registry, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.12.1/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..30299022c5 --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/meta_yml.md @@ -0,0 +1,28 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module: NFCoreComponent){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/meta-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. + +If the module has inputs or outputs, they are expected to be +formatted as: + +..code-block:: +: tuple val(foo) path(bar) +val foo +path foo + +or permutations of the above. + +- **Parameters:** + - **module_lint_object** (_ComponentLint_) – The lint object for the module + - **module** (_NFCoreComponent_) – The module to lint diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/module_changes.md b/src/content/tools/docs/2.12.1/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.12.1/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/module_patch.md b/src/content/tools/docs/2.12.1/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..5d38a40d31 --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreComponent){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/module_tests.md b/src/content/tools/docs/2.12.1/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..df2872f29e --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/module_tests.md @@ -0,0 +1,8 @@ +# module_tests + +#### `ModuleLint.module_tests(module: NFCoreComponent){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/module_todos.md b/src/content/tools/docs/2.12.1/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.12.1/module_lint_tests/module_version.md b/src/content/tools/docs/2.12.1/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.12.1/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..aec43b98fd --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `NXF_VER` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + NXF_VER: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `NXF_VER` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b5b230ab71 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,116 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +pipeline_template.yml # saving information in .nf-core.yml +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +Files that _must not_ be present if a certain entry is present in `nextflow.config`: + +```bash +lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ccb1541ae3 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,55 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..c8cf0fe7b1 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/index.md @@ -0,0 +1,44 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [system_exit](system_exit) + - [`PipelineLint.system_exit()`](system_exit#nf_core.lint.PipelineLint.system_exit) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..c6d01e3bd3 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,25 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` + +:::note +You can choose to ignore this lint tests by editing the file called +`.nf-core.yml` in the root of your pipeline and setting the test to false: +.. code-block:: yaml +::: + +> lint: +> : merge_markers: False + +To disable this test only for specific files, you can specify a list of file paths to ignore. +For example, to ignore a pdf you added to the docs: +.. code-block:: yaml + +> lint: +> : merge_markers: +> : - docs/my_pdf.pdf diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..ba2ade27c8 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,114 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.validationShowHiddenParams` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.validationSchemaIgnoreParams` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` + +**The configuration should contain the following or the test will fail:** + +- A `test` configuration profile should exist. + +**The default values in \`\`nextflow.config\`\` should match the default values defined in the \`\`nextflow_schema.json\`\`.** + +:::note +You can choose to ignore tests for the default value of an specific parameter +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config parameters that should be ignored. For example to ignore the default value for the input parameter: + +```yaml +lint: + nextflow_config: + - config_defaults: + - params.input +``` + +::: diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..37b9794794 --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,37 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. + +:::note +You can choose to ignore lint test tests by editing the file called +`.nf-core.yml` in the root of your pipeline and setting the test to false: + +```yaml +lint: + template_strings: False +``` + +::: + +To disable this test only for specific files, you can specify a list of file paths to ignore. +For example, to ignore a pdf you added to the docs: + +```yaml +lint: + template_strings: + - docs/my_pdf.pdf +``` diff --git a/src/content/tools/docs/2.12.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.12.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.12.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.12.1/subworkflow_lint_tests/index.md b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/index.md new file mode 100644 index 0000000000..d5061028dc --- /dev/null +++ b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/index.md @@ -0,0 +1,16 @@ +# Subworkflow lint tests + +# Tests: + +- [main_nf](main_nf) + - [`SubworkflowLint.main_nf()`](main_nf#nf_core.subworkflows.lint.SubworkflowLint.main_nf) +- [meta_yml](meta_yml) + - [`SubworkflowLint.meta_yml()`](meta_yml#nf_core.subworkflows.lint.SubworkflowLint.meta_yml) +- [subworkflow_changes](subworkflow_changes) + - [`SubworkflowLint.subworkflow_changes()`](subworkflow_changes#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_changes) +- [subworkflow_tests](subworkflow_tests) + - [`SubworkflowLint.subworkflow_tests()`](subworkflow_tests#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_tests) +- [subworkflow_todos](subworkflow_todos) + - [`SubworkflowLint.subworkflow_todos()`](subworkflow_todos#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_todos) +- [subworkflow_version](subworkflow_version) + - [`SubworkflowLint.subworkflow_version()`](subworkflow_version#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_version) diff --git a/src/content/tools/docs/2.12.1/subworkflow_lint_tests/main_nf.md b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/main_nf.md new file mode 100644 index 0000000000..1280380868 --- /dev/null +++ b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/main_nf.md @@ -0,0 +1,16 @@ +# main_nf + +#### `SubworkflowLint.main_nf(subworkflow){:python}` + +Lint a `main.nf` subworkflow file + +Can also be used to lint local subworkflow files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- A subworkflow SHOULD import at least two modules +- All included modules or subworkflows are used and their names are used for versions.yml +- The workflow name is all capital letters +- The subworkflow emits a software version diff --git a/src/content/tools/docs/2.12.1/subworkflow_lint_tests/meta_yml.md b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/meta_yml.md new file mode 100644 index 0000000000..2ab290d54b --- /dev/null +++ b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/meta_yml.md @@ -0,0 +1,17 @@ +# meta_yml + +#### `SubworkflowLint.meta_yml(subworkflow){:python}` + +Lint a `meta.yml` file + +The lint test checks that the subworkflow has +a `meta.yml` file and that it follows the +JSON schema defined in the `subworkflows/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the subworkflow name +and subworkflow input is consistent between the +`meta.yml` and the `main.nf`. + +Checks that all input and output channels are specified in `meta.yml`. +Checks that all included components in `main.nf` are specified in `meta.yml`. diff --git a/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_changes.md b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_changes.md new file mode 100644 index 0000000000..cda417f1b6 --- /dev/null +++ b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_changes.md @@ -0,0 +1,14 @@ +# subworkflow_changes + +#### `SubworkflowLint.subworkflow_changes(subworkflow){:python}` + +Checks whether installed nf-core subworkflow have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every subworkflow +and compares them to the local copies + +If the subworkflow has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_tests.md b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_tests.md new file mode 100644 index 0000000000..240405567b --- /dev/null +++ b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_tests.md @@ -0,0 +1,10 @@ +# subworkflow_tests + +#### `SubworkflowLint.subworkflow_tests(subworkflow: NFCoreComponent){:python}` + +Lint the tests of a subworkflow in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. + +Additionally, hecks that all included components in test `main.nf` are specified in `test.yml` diff --git a/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_todos.md b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_todos.md new file mode 100644 index 0000000000..e17f92c01d --- /dev/null +++ b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_todos.md @@ -0,0 +1,28 @@ +# subworkflow_todos + +#### `SubworkflowLint.subworkflow_todos(subworkflow){:python}` + +Look for TODO statements in the subworkflow files + +The nf-core subworkflow template contains a number of comment lines to help developers +of new subworkflow know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the subworkflows and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_version.md b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_version.md new file mode 100644 index 0000000000..9b398909f7 --- /dev/null +++ b/src/content/tools/docs/2.12.1/subworkflow_lint_tests/subworkflow_version.md @@ -0,0 +1,9 @@ +# subworkflow_version + +#### `SubworkflowLint.subworkflow_version(subworkflow){:python}` + +Verifies that the subworkflow has a version specified in the `modules.json` file + +It checks whether the subworkflow has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the subworkflow available. diff --git a/src/content/tools/docs/2.12/api/bump_version.md b/src/content/tools/docs/2.12/api/bump_version.md new file mode 100644 index 0000000000..107f374a18 --- /dev/null +++ b/src/content/tools/docs/2.12/api/bump_version.md @@ -0,0 +1,34 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename: str | Path, pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, patterns: List[Tuple[str, str]]){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.12/api/create.md b/src/content/tools/docs/2.12/api/create.md new file mode 100644 index 0000000000..048529adb0 --- /dev/null +++ b/src/content/tools/docs/2.12/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain, pipeline_dir){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. + - **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + - **plain** (_bool_) – If true the pipeline template will be initialized plain, without customisation. + - **pipeline_dir** (_str_) – Path to the pipeline directory. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + **template_areas** (_list_) – List of available template areas to skip. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.12/api/download.md b/src/content/tools/docs/2.12/api/download.md new file mode 100644 index 0000000000..1a99ec6506 --- /dev/null +++ b/src/content/tools/docs/2.12/api/download.md @@ -0,0 +1,309 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`exception{:python}`_`nf_core.download.ContainerError(container, registry, address, absolute_URI, out_path, singularity_command, error_msg){:python}` + +Bases: `Exception` + +A class of errors related to pulling containers with Singularity/Apptainer + +#### `undefined{:python}` ImageExistsError(error_log) + +Bases: `FileExistsError` + +Image already exists in cache/output directory. + +#### `undefined{:python}` ImageNotFoundError(error_log) + +Bases: `FileNotFoundError` + +The image can not be found in the registry + +#### `undefined{:python}` InvalidTagError(error_log) + +Bases: `AttributeError` + +Image and registry are valid, but the (version) tag is not + +#### `undefined{:python}` OtherError(error_log) + +Bases: `RuntimeError` + +Undefined error with the container + +#### `undefined{:python}` RegistryNotFoundError(error_log) + +Bases: `ConnectionRefusedError` + +The specified registry does not resolve to a valid IP address + +### _`exception{:python}`_`nf_core.download.DownloadError{:python}` + +Bases: `RuntimeError` + +A custom exception that is raised when nf-core download encounters a problem that we already took into consideration. +In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, tower=False, download_configuration=None, container_system=None, container_library=None, container_cache_utilisation=None, container_cache_index=None, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_List_\*\[**str**]\*) – The workflow revision to download, like 1.0. Defaults to None. + - **container** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **tower** (_bool_) – Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(revision, wf_sha, download_url){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `download_workflow_static(){:python}` + +Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner. + +#### `download_workflow_tower(location=None){:python}` + +Create a bare-cloned git repository of the workflow, so it can be launched with tw launch as pipeline + +#### `find_container_images(workflow_directory){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic, +but not always, e.g. not for differentialabundance 1.2.0 + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw matches that will likely need to be cleaned. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(current_revision=''){:python}` + +Loop through container names and download Singularity images + +#### `prioritize_direct_download(container_list){:python}` + +Helper function that takes a list of container images (URLs and Docker URIs), +eliminates all Docker URIs for which also a URL is contained and returns the +cleaned and also deduplicated list. + +Conceptually, this works like so: + +Everything after the last Slash should be identical, e.g. “scanpy:1.7.2–pyhdfd78af_0” in +\[‘https://depot.galaxyproject.org/singularity/scanpy:1.7.2–pyhdfd78af\_0’, ‘biocontainers/scanpy:1.7.2–pyhdfd78af_0’] + +re.sub(‘.\*/(.\*)’,’1’,c) will drop everything up to the last slash from c (container_id) + +d.get(k:=re.sub(‘.\*/(.\*)’,’1’,c),’’) assigns the truncated string to k (key) and gets the +corresponding value from the dict if present or else defaults to “”. + +If the regex pattern matches, the original container_id will be assigned to the dict with the k key. +r”^$|(?!^http)” matches an empty string (we didn’t have it in the dict yet and want to keep it in either case) or +any string that does not start with http. Because if our current dict value already starts with http, +we want to keep it and not replace with with whatever we have now (which might be the Docker URI). + +A regex that matches http, r”^$|^http” could thus be used to prioritize the Docker URIs over http Downloads + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_config_inclusion(){:python}` + +Prompt for inclusion of institutional configurations + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch +Prompt user for revision tag if ‘–revision’ was not set +If –tower is specified, allow to select multiple revisions +Also the static download allows for multiple revisions, but +we do not prompt this option interactively. + +#### `prompt_singularity_cachedir_creation(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `prompt_singularity_cachedir_remote(){:python}` + +Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR + +#### `prompt_singularity_cachedir_utilization(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `read_remote_containers(){:python}` + +Reads the file specified as index for the remote Singularity cache dir + +#### `rectify_raw_container_matches(raw_findings){:python}` + +Helper function to rectify the raw extracted container matches into fully qualified container names. +If multiple containers are found, any prefixed with http for direct download is prioritized + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘biocontainers/fastqc:0.11.9–0’ }” + +Later DSL2, variable is being used: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity\_pull\_docker\_container ? +: “/${container_id}” : +“quay.io/biocontainers/${container_id}” }”
      +container_id = ‘mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0’ + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, library, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. + - **library** (_list_ _of_ _str_) – A list of libraries to try for pulling the image. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(revision_dirname){:python}` + +Edit the downloaded nextflow.config file to use the local config files + +### _`class{:python}`_`nf_core.download.WorkflowRepo(remote_url, revision, commit, location=None, hide_progress=False, in_cache=True){:python}` + +Bases: `SyncedRepo` + +An object to store details about a locally cached workflow repository. + +Important Attributes: +: fullname: The full name of the repository, `nf-core/{self.pipelinename}`. +local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to `$HOME/.cache/nf-core/nf-core/{self.pipeline}`. + +#### `access(){:python}` + +#### `bare_clone(destination){:python}` + +#### `checkout(commit){:python}` + +Checks out the repository at the requested commit + +- **Parameters:** + **commit** (_str_) – Git SHA of the commit + +#### `get_remote_branches(remote_url){:python}` + +Get all branches from a remote repository + +- **Parameters:** + **remote_url** (_str_) – The git url to the remote repository +- **Returns:** + All branches found in the remote +- **Return type:** + (set\[str]) + +#### `retry_setup_local_repo(skip_confirm=False){:python}` + +#### `setup_local_repo(remote, location=None, in_cache=True){:python}` + +Sets up the local git repository. If the repository has been cloned previously, it +returns a git.Repo object of that clone. Otherwise it tries to clone the repository from +the provided remote URL and returns a git.Repo of the new clone. + +- **Parameters:** + - **remote** (_str_) – git url of remote + - **location** (_Path_) – location where the clone should be created/cached. + - **in_cache** (_bool_\*,\* _optional_) – Whether to clone the repository from the cache. Defaults to False. + +Sets self.repo + +#### `tidy_tags_and_branches(){:python}` + +Function to delete all tags and branches that are not of interest to the downloader. +This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + +However, due to local caching, the downloader might also want access to revisions that had been deleted before. +In that case, don’t bother with re-adding the tags and rather download anew from Github. diff --git a/src/content/tools/docs/2.12/api/index.md b/src/content/tools/docs/2.12/api/index.md new file mode 100644 index 0000000000..340d966970 --- /dev/null +++ b/src/content/tools/docs/2.12/api/index.md @@ -0,0 +1,79 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`ContainerError`](download#nf_core.download.ContainerError) + - [`DownloadError`](download#nf_core.download.DownloadError) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) + - [`WorkflowRepo`](download#nf_core.download.WorkflowRepo) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.params_file](params-file) + - [`ParamsFileBuilder`](params-file#nf_core.params_file.ParamsFileBuilder) + - [`_print_wrapped()`](params-file#nf_core.params_file._print_wrapped) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestExceptionError`](sync#nf_core.sync.PullRequestExceptionError) + - [`SyncExceptionError`](sync#nf_core.sync.SyncExceptionError) +- [nf_core.utils](utils) + - [`GitHubAPISession`](utils#nf_core.utils.GitHubAPISession) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`SingularityCacheFilePathValidator`](utils#nf_core.utils.SingularityCacheFilePathValidator) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_remote_version()`](utils#nf_core.utils.fetch_remote_version) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nested_delitem()`](utils#nf_core.utils.nested_delitem) + - [`nested_setitem()`](utils#nf_core.utils.nested_setitem) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`run_cmd()`](utils#nf_core.utils.run_cmd) + - [`set_wd()`](utils#nf_core.utils.set_wd) + - [`setup_nfcore_cachedir()`](utils#nf_core.utils.setup_nfcore_cachedir) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.12/api/launch.md b/src/content/tools/docs/2.12/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.12/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.12/api/licences.md b/src/content/tools/docs/2.12/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.12/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.12/api/lint.md b/src/content/tools/docs/2.12/api/lint.md new file mode 100644 index 0000000000..d1c0d168bd --- /dev/null +++ b/src/content/tools/docs/2.12/api/lint.md @@ -0,0 +1,134 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + An object of type `ComponentLint` that contains all the linting results for the modules. + An object of type `ComponentLint` that contains all the linting results for the subworkflows. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.12/api/list.md b/src/content/tools/docs/2.12/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.12/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.12/api/modules.md b/src/content/tools/docs/2.12/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.12/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.12/api/params-file.md b/src/content/tools/docs/2.12/api/params-file.md new file mode 100644 index 0000000000..6cef34b10a --- /dev/null +++ b/src/content/tools/docs/2.12/api/params-file.md @@ -0,0 +1,84 @@ +# nf_core.params_file + +Create a YAML parameter file + +### _`class{:python}`_`nf_core.params_file.ParamsFileBuilder(pipeline=None, revision=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline. + +- **Parameters:** + - **pipeline** (_str_\*,\* _optional_) – Path to a local pipeline path or a remote pipeline. + - **revision** (_str_\*,\* _optional_) – Revision of the pipeline to use. + +#### `format_group(definition, show_hidden=False){:python}` + +Format a group of parameters of the schema as commented YAML. + +- **Parameters:** + - **definition** (_dict_) – Definition of the group from the schema + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for a group +- **Return type:** + str + +#### `format_param(name, properties, required_properties=(), show_hidden=False){:python}` + +Format a single parameter of the schema as commented YAML + +- **Parameters:** + - **name** (_str_) – Name of the parameter + - **properties** (_dict_) – Properties of the parameter + - **required_properties** (_list_) – List of required properties + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Section of a params-file.yml for given parameter + None: + > If the parameter is skipped because it is hidden and + > show_hidden is not set +- **Return type:** + str + +#### `generate_params_file(show_hidden=False){:python}` + +Generate the contents of a parameter template file. + +Assumes the pipeline has been fetched (if remote) and the schema loaded. + +- **Parameters:** + **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for the pipeline schema +- **Return type:** + str + +#### `get_pipeline(){:python}` + +Prompt the user for a pipeline name and get the schema + +#### `write_params_file(output_fn='nf-params.yaml', show_hidden=False, force=False){:python}` + +Build a template file for the pipeline schema. + +- **Parameters:** + - **output_fn** (_str_\*,\* _optional_) – Filename to write the template to. + - **show_hidden** (_bool_\*,\* _optional_) – Include parameters marked as hidden in the output + - **force** (_bool_\*,\* _optional_) – Whether to overwrite existing output file. +- **Returns:** + True if the template was written successfully, False otherwise +- **Return type:** + bool + +### `nf_core.params_file._print_wrapped(text, fill_char='-', mode='both', width=80, indent=0, drop_whitespace=True){:python}` + +Helper function to format text for the params-file template. + +- **Parameters:** + - **text** (_str_) – Text to print + - **fill_char** (_str_\*,\* _optional_) – Character to use for creating dividers. Defaults to ‘-‘. + - **mode** (_str_\*,\* _optional_) – Where to place dividers. Defaults to “both”. + - **width** (_int_\*,\* _optional_) – Maximum line-width of the output text. Defaults to 80. + - **indent** (_int_\*,\* _optional_) – Number of spaces to indent the text. Defaults to 0. + - **drop_whitespace** (_bool_\*,\* _optional_) – Whether to drop whitespace from the start and end of lines. diff --git a/src/content/tools/docs/2.12/api/schema.md b/src/content/tools/docs/2.12/api/schema.md new file mode 100644 index 0000000000..3ce5414c20 --- /dev/null +++ b/src/content/tools/docs/2.12/api/schema.md @@ -0,0 +1,166 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema +Update defaults if they have changed + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.12/api/subworkflows.md b/src/content/tools/docs/2.12/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.12/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.12/api/sync.md b/src/content/tools/docs/2.12/api/sync.md new file mode 100644 index 0000000000..bc7034cf1c --- /dev/null +++ b/src/content/tools/docs/2.12/api/sync.md @@ -0,0 +1,152 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. DEPRECATED + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestExceptionError{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncExceptionError{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.12/api/utils.md b/src/content/tools/docs/2.12/api/utils.md new file mode 100644 index 0000000000..4b4cae4ea6 --- /dev/null +++ b/src/content/tools/docs/2.12/api/utils.md @@ -0,0 +1,408 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHubAPISession{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### _`class{:python}`_`nf_core.utils.SingularityCacheFilePathValidator{:python}` + +Bases: `Validator` + +Validator for file path specified as –singularity-cache-index argument in nf-core download + +#### `_abc_impl{:python}`_= <\_abc.\_abc_data object>_ + +#### `validate(value){:python}` + +Validate the input. +If invalid, this should raise a `ValidationError`. + +- **Parameters:** + **document** – `Document` instance. + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_remote_version(source_url){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory: str | Path = '.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nested_delitem(d, keys){:python}` + +Deletes a key from a nested dictionary + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse, deleting the final one + +### `nf_core.utils.nested_setitem(d, keys, value){:python}` + +Sets the value in a nested dict using a list of keys to traverse + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse + - **value** (_Any_) – The value to be set for the last key in the chain + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API + - **multiple** (_bool_) – Allow selection of multiple releases & branches (for Tower) +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.run_cmd(executable: str, cmd: str){:python}` + +Run a specified command and capture the output. Handle errors nicely. + +### `nf_core.utils.set_wd(path: Path){:python}` + +Sets the working directory for this context. + +- **Parameters:** + **path** (_Path_) – Path to the working directory to be used inside this context. + +### `nf_core.utils.setup_nfcore_cachedir(cache_fn: str | Path){:python}` + +Sets up local caching for caching files between sessions. + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.12/index.md b/src/content/tools/docs/2.12/index.md new file mode 100644 index 0000000000..636ffaa787 --- /dev/null +++ b/src/content/tools/docs/2.12/index.md @@ -0,0 +1,12 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [Subworkflow code lint tests](subworkflow_lint_tests/index) (run by `nf-core subworkflows lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.12/module_lint_tests/index.md b/src/content/tools/docs/2.12/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.12/module_lint_tests/main_nf.md b/src/content/tools/docs/2.12/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..aa42cef6d7 --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, registry, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.12/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.12/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..30299022c5 --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/meta_yml.md @@ -0,0 +1,28 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module: NFCoreComponent){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/meta-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. + +If the module has inputs or outputs, they are expected to be +formatted as: + +..code-block:: +: tuple val(foo) path(bar) +val foo +path foo + +or permutations of the above. + +- **Parameters:** + - **module_lint_object** (_ComponentLint_) – The lint object for the module + - **module** (_NFCoreComponent_) – The module to lint diff --git a/src/content/tools/docs/2.12/module_lint_tests/module_changes.md b/src/content/tools/docs/2.12/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.12/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.12/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.12/module_lint_tests/module_patch.md b/src/content/tools/docs/2.12/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..5d38a40d31 --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreComponent){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.12/module_lint_tests/module_tests.md b/src/content/tools/docs/2.12/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..df2872f29e --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/module_tests.md @@ -0,0 +1,8 @@ +# module_tests + +#### `ModuleLint.module_tests(module: NFCoreComponent){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. diff --git a/src/content/tools/docs/2.12/module_lint_tests/module_todos.md b/src/content/tools/docs/2.12/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.12/module_lint_tests/module_version.md b/src/content/tools/docs/2.12/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.12/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..aec43b98fd --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `NXF_VER` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + NXF_VER: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `NXF_VER` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.12/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b5b230ab71 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/files_exist.md @@ -0,0 +1,116 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +pipeline_template.yml # saving information in .nf-core.yml +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +Files that _must not_ be present if a certain entry is present in `nextflow.config`: + +```bash +lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.12/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ec6119fbb7 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,61 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +Files that need to be there or not based on a entry in nextflow config: + +```default +lib/nfcore_external_java_deps.jar # if config doesn't mention nf-validation +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/index.md b/src/content/tools/docs/2.12/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..c8cf0fe7b1 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/index.md @@ -0,0 +1,44 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [system_exit](system_exit) + - [`PipelineLint.system_exit()`](system_exit#nf_core.lint.PipelineLint.system_exit) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.12/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..c6d01e3bd3 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,25 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` + +:::note +You can choose to ignore this lint tests by editing the file called +`.nf-core.yml` in the root of your pipeline and setting the test to false: +.. code-block:: yaml +::: + +> lint: +> : merge_markers: False + +To disable this test only for specific files, you can specify a list of file paths to ignore. +For example, to ignore a pdf you added to the docs: +.. code-block:: yaml + +> lint: +> : merge_markers: +> : - docs/my_pdf.pdf diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.12/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.12/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.12/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.12/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..ba2ade27c8 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,114 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.validationShowHiddenParams` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.validationSchemaIgnoreParams` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` + +**The configuration should contain the following or the test will fail:** + +- A `test` configuration profile should exist. + +**The default values in \`\`nextflow.config\`\` should match the default values defined in the \`\`nextflow_schema.json\`\`.** + +:::note +You can choose to ignore tests for the default value of an specific parameter +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config parameters that should be ignored. For example to ignore the default value for the input parameter: + +```yaml +lint: + nextflow_config: + - config_defaults: + - params.input +``` + +::: diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.12/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.12/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.12/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.12/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.12/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.12/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/2.12/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.12/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..37b9794794 --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/template_strings.md @@ -0,0 +1,37 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. + +:::note +You can choose to ignore lint test tests by editing the file called +`.nf-core.yml` in the root of your pipeline and setting the test to false: + +```yaml +lint: + template_strings: False +``` + +::: + +To disable this test only for specific files, you can specify a list of file paths to ignore. +For example, to ignore a pdf you added to the docs: + +```yaml +lint: + template_strings: + - docs/my_pdf.pdf +``` diff --git a/src/content/tools/docs/2.12/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.12/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.12/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.12/subworkflow_lint_tests/index.md b/src/content/tools/docs/2.12/subworkflow_lint_tests/index.md new file mode 100644 index 0000000000..d5061028dc --- /dev/null +++ b/src/content/tools/docs/2.12/subworkflow_lint_tests/index.md @@ -0,0 +1,16 @@ +# Subworkflow lint tests + +# Tests: + +- [main_nf](main_nf) + - [`SubworkflowLint.main_nf()`](main_nf#nf_core.subworkflows.lint.SubworkflowLint.main_nf) +- [meta_yml](meta_yml) + - [`SubworkflowLint.meta_yml()`](meta_yml#nf_core.subworkflows.lint.SubworkflowLint.meta_yml) +- [subworkflow_changes](subworkflow_changes) + - [`SubworkflowLint.subworkflow_changes()`](subworkflow_changes#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_changes) +- [subworkflow_tests](subworkflow_tests) + - [`SubworkflowLint.subworkflow_tests()`](subworkflow_tests#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_tests) +- [subworkflow_todos](subworkflow_todos) + - [`SubworkflowLint.subworkflow_todos()`](subworkflow_todos#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_todos) +- [subworkflow_version](subworkflow_version) + - [`SubworkflowLint.subworkflow_version()`](subworkflow_version#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_version) diff --git a/src/content/tools/docs/2.12/subworkflow_lint_tests/main_nf.md b/src/content/tools/docs/2.12/subworkflow_lint_tests/main_nf.md new file mode 100644 index 0000000000..1280380868 --- /dev/null +++ b/src/content/tools/docs/2.12/subworkflow_lint_tests/main_nf.md @@ -0,0 +1,16 @@ +# main_nf + +#### `SubworkflowLint.main_nf(subworkflow){:python}` + +Lint a `main.nf` subworkflow file + +Can also be used to lint local subworkflow files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- A subworkflow SHOULD import at least two modules +- All included modules or subworkflows are used and their names are used for versions.yml +- The workflow name is all capital letters +- The subworkflow emits a software version diff --git a/src/content/tools/docs/2.12/subworkflow_lint_tests/meta_yml.md b/src/content/tools/docs/2.12/subworkflow_lint_tests/meta_yml.md new file mode 100644 index 0000000000..2ab290d54b --- /dev/null +++ b/src/content/tools/docs/2.12/subworkflow_lint_tests/meta_yml.md @@ -0,0 +1,17 @@ +# meta_yml + +#### `SubworkflowLint.meta_yml(subworkflow){:python}` + +Lint a `meta.yml` file + +The lint test checks that the subworkflow has +a `meta.yml` file and that it follows the +JSON schema defined in the `subworkflows/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the subworkflow name +and subworkflow input is consistent between the +`meta.yml` and the `main.nf`. + +Checks that all input and output channels are specified in `meta.yml`. +Checks that all included components in `main.nf` are specified in `meta.yml`. diff --git a/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_changes.md b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_changes.md new file mode 100644 index 0000000000..cda417f1b6 --- /dev/null +++ b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_changes.md @@ -0,0 +1,14 @@ +# subworkflow_changes + +#### `SubworkflowLint.subworkflow_changes(subworkflow){:python}` + +Checks whether installed nf-core subworkflow have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every subworkflow +and compares them to the local copies + +If the subworkflow has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_tests.md b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_tests.md new file mode 100644 index 0000000000..240405567b --- /dev/null +++ b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_tests.md @@ -0,0 +1,10 @@ +# subworkflow_tests + +#### `SubworkflowLint.subworkflow_tests(subworkflow: NFCoreComponent){:python}` + +Lint the tests of a subworkflow in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. + +Additionally, hecks that all included components in test `main.nf` are specified in `test.yml` diff --git a/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_todos.md b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_todos.md new file mode 100644 index 0000000000..e17f92c01d --- /dev/null +++ b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_todos.md @@ -0,0 +1,28 @@ +# subworkflow_todos + +#### `SubworkflowLint.subworkflow_todos(subworkflow){:python}` + +Look for TODO statements in the subworkflow files + +The nf-core subworkflow template contains a number of comment lines to help developers +of new subworkflow know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the subworkflows and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_version.md b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_version.md new file mode 100644 index 0000000000..9b398909f7 --- /dev/null +++ b/src/content/tools/docs/2.12/subworkflow_lint_tests/subworkflow_version.md @@ -0,0 +1,9 @@ +# subworkflow_version + +#### `SubworkflowLint.subworkflow_version(subworkflow){:python}` + +Verifies that the subworkflow has a version specified in the `modules.json` file + +It checks whether the subworkflow has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the subworkflow available. diff --git a/src/content/tools/docs/2.2/api/bump_version.md b/src/content/tools/docs/2.2/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.2/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.2/api/create.md b/src/content/tools/docs/2.2/api/create.md new file mode 100644 index 0000000000..302bfd36a6 --- /dev/null +++ b/src/content/tools/docs/2.2/api/create.md @@ -0,0 +1,36 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.2/api/download.md b/src/content/tools/docs/2.2/api/download.md new file mode 100644 index 0000000000..a04af4c422 --- /dev/null +++ b/src/content/tools/docs/2.2/api/download.md @@ -0,0 +1,142 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.2/api/index.md b/src/content/tools/docs/2.2/api/index.md new file mode 100644 index 0000000000..8f79de1474 --- /dev/null +++ b/src/content/tools/docs/2.2/api/index.md @@ -0,0 +1,56 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.2/api/launch.md b/src/content/tools/docs/2.2/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.2/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.2/api/licences.md b/src/content/tools/docs/2.2/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.2/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.2/api/lint.md b/src/content/tools/docs/2.2/api/lint.md new file mode 100644 index 0000000000..58da46f58a --- /dev/null +++ b/src/content/tools/docs/2.2/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.2/api/list.md b/src/content/tools/docs/2.2/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.2/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.2/api/modules.md b/src/content/tools/docs/2.2/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.2/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.2/api/schema.md b/src/content/tools/docs/2.2/api/schema.md new file mode 100644 index 0000000000..1212a43977 --- /dev/null +++ b/src/content/tools/docs/2.2/api/schema.md @@ -0,0 +1,121 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.2/api/sync.md b/src/content/tools/docs/2.2/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.2/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.2/api/utils.md b/src/content/tools/docs/2.2/api/utils.md new file mode 100644 index 0000000000..93d52b9ab7 --- /dev/null +++ b/src/content/tools/docs/2.2/api/utils.md @@ -0,0 +1,269 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.2/index.md b/src/content/tools/docs/2.2/index.md new file mode 100644 index 0000000000..c3d07a7628 --- /dev/null +++ b/src/content/tools/docs/2.2/index.md @@ -0,0 +1,13 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.2/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..f25b5c62b4 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/files_exist.md @@ -0,0 +1,87 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yaml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.2/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..594dcbfe50 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,56 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +assets/multiqc_config.yaml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/index.md b/src/content/tools/docs/2.2/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..d73f5fbd15 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/index.md @@ -0,0 +1,38 @@ +# Pipline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.2/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.2/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..40c566e813 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version git_sha hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.2/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.2/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.2/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..e8a73d1663 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.2/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.2/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.2/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.2/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.2/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.2/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.2/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.2/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.3.1/api/bump_version.md b/src/content/tools/docs/2.3.1/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.3.1/api/create.md b/src/content/tools/docs/2.3.1/api/create.md new file mode 100644 index 0000000000..1a80c6b013 --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/create.md @@ -0,0 +1,40 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.3.1/api/download.md b/src/content/tools/docs/2.3.1/api/download.md new file mode 100644 index 0000000000..2cb1eb33c3 --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/download.md @@ -0,0 +1,162 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.3.1/api/index.md b/src/content/tools/docs/2.3.1/api/index.md new file mode 100644 index 0000000000..8f79de1474 --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/index.md @@ -0,0 +1,56 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.3.1/api/launch.md b/src/content/tools/docs/2.3.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.3.1/api/licences.md b/src/content/tools/docs/2.3.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.3.1/api/lint.md b/src/content/tools/docs/2.3.1/api/lint.md new file mode 100644 index 0000000000..cf1c641287 --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.3.1/api/list.md b/src/content/tools/docs/2.3.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.3.1/api/modules.md b/src/content/tools/docs/2.3.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.3.1/api/schema.md b/src/content/tools/docs/2.3.1/api/schema.md new file mode 100644 index 0000000000..52b361a2be --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/schema.md @@ -0,0 +1,133 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=['parameter', 'description', 'type,', 'default', 'required', 'hidden']){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.3.1/api/sync.md b/src/content/tools/docs/2.3.1/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.3.1/api/utils.md b/src/content/tools/docs/2.3.1/api/utils.md new file mode 100644 index 0000000000..93d52b9ab7 --- /dev/null +++ b/src/content/tools/docs/2.3.1/api/utils.md @@ -0,0 +1,269 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.3.1/index.md b/src/content/tools/docs/2.3.1/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.3.1/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.3.1/module_lint_tests/index.md b/src/content/tools/docs/2.3.1/module_lint_tests/index.md new file mode 100644 index 0000000000..457d4b460a --- /dev/null +++ b/src/content/tools/docs/2.3.1/module_lint_tests/index.md @@ -0,0 +1,12 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) diff --git a/src/content/tools/docs/2.3.1/module_lint_tests/main_nf.md b/src/content/tools/docs/2.3.1/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..0d9a16a731 --- /dev/null +++ b/src/content/tools/docs/2.3.1/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.3.1/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.3.1/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.3.1/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.3.1/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.3.1/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.3.1/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.3.1/module_lint_tests/module_todos.md b/src/content/tools/docs/2.3.1/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.3.1/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..13a59f9f5e --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,89 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..24b64a7fe0 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,54 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..5a0789075d --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/index.md @@ -0,0 +1,40 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.3.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.3.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.3.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.3.2/api/bump_version.md b/src/content/tools/docs/2.3.2/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.3.2/api/create.md b/src/content/tools/docs/2.3.2/api/create.md new file mode 100644 index 0000000000..1a80c6b013 --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/create.md @@ -0,0 +1,40 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.3.2/api/download.md b/src/content/tools/docs/2.3.2/api/download.md new file mode 100644 index 0000000000..2cb1eb33c3 --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/download.md @@ -0,0 +1,162 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.3.2/api/index.md b/src/content/tools/docs/2.3.2/api/index.md new file mode 100644 index 0000000000..8f79de1474 --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/index.md @@ -0,0 +1,56 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.3.2/api/launch.md b/src/content/tools/docs/2.3.2/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.3.2/api/licences.md b/src/content/tools/docs/2.3.2/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.3.2/api/lint.md b/src/content/tools/docs/2.3.2/api/lint.md new file mode 100644 index 0000000000..cf1c641287 --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.3.2/api/list.md b/src/content/tools/docs/2.3.2/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.3.2/api/modules.md b/src/content/tools/docs/2.3.2/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.3.2/api/schema.md b/src/content/tools/docs/2.3.2/api/schema.md new file mode 100644 index 0000000000..52b361a2be --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/schema.md @@ -0,0 +1,133 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=['parameter', 'description', 'type,', 'default', 'required', 'hidden']){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.3.2/api/sync.md b/src/content/tools/docs/2.3.2/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.3.2/api/utils.md b/src/content/tools/docs/2.3.2/api/utils.md new file mode 100644 index 0000000000..93d52b9ab7 --- /dev/null +++ b/src/content/tools/docs/2.3.2/api/utils.md @@ -0,0 +1,269 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.3.2/index.md b/src/content/tools/docs/2.3.2/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.3.2/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.3.2/module_lint_tests/index.md b/src/content/tools/docs/2.3.2/module_lint_tests/index.md new file mode 100644 index 0000000000..457d4b460a --- /dev/null +++ b/src/content/tools/docs/2.3.2/module_lint_tests/index.md @@ -0,0 +1,12 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) diff --git a/src/content/tools/docs/2.3.2/module_lint_tests/main_nf.md b/src/content/tools/docs/2.3.2/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..0d9a16a731 --- /dev/null +++ b/src/content/tools/docs/2.3.2/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.3.2/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.3.2/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.3.2/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.3.2/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.3.2/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.3.2/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.3.2/module_lint_tests/module_todos.md b/src/content/tools/docs/2.3.2/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.3.2/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..766b53eeef --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/files_exist.md @@ -0,0 +1,91 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..24b64a7fe0 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,54 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/index.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..5a0789075d --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/index.md @@ -0,0 +1,40 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.3.2/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.3.2/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.3.2/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.3/api/bump_version.md b/src/content/tools/docs/2.3/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.3/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.3/api/create.md b/src/content/tools/docs/2.3/api/create.md new file mode 100644 index 0000000000..1a80c6b013 --- /dev/null +++ b/src/content/tools/docs/2.3/api/create.md @@ -0,0 +1,40 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.3/api/download.md b/src/content/tools/docs/2.3/api/download.md new file mode 100644 index 0000000000..2cb1eb33c3 --- /dev/null +++ b/src/content/tools/docs/2.3/api/download.md @@ -0,0 +1,162 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.3/api/index.md b/src/content/tools/docs/2.3/api/index.md new file mode 100644 index 0000000000..8f79de1474 --- /dev/null +++ b/src/content/tools/docs/2.3/api/index.md @@ -0,0 +1,56 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`github_api_auto_auth()`](utils#nf_core.utils.github_api_auto_auth) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.3/api/launch.md b/src/content/tools/docs/2.3/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.3/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.3/api/licences.md b/src/content/tools/docs/2.3/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.3/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.3/api/lint.md b/src/content/tools/docs/2.3/api/lint.md new file mode 100644 index 0000000000..58da46f58a --- /dev/null +++ b/src/content/tools/docs/2.3/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +#### `SEE ALSO{:python}` + +See the [Lint Tests](../lint_tests/index.html) docs for information about specific linting functions. + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.3/api/list.md b/src/content/tools/docs/2.3/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.3/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.3/api/modules.md b/src/content/tools/docs/2.3/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.3/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.3/api/schema.md b/src/content/tools/docs/2.3/api/schema.md new file mode 100644 index 0000000000..52b361a2be --- /dev/null +++ b/src/content/tools/docs/2.3/api/schema.md @@ -0,0 +1,133 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=['parameter', 'description', 'type,', 'default', 'required', 'hidden']){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.3/api/sync.md b/src/content/tools/docs/2.3/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.3/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.3/api/utils.md b/src/content/tools/docs/2.3/api/utils.md new file mode 100644 index 0000000000..93d52b9ab7 --- /dev/null +++ b/src/content/tools/docs/2.3/api/utils.md @@ -0,0 +1,269 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.github_api_auto_auth(){:python}` + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .nfcore_cache subdir. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.3/index.md b/src/content/tools/docs/2.3/index.md new file mode 100644 index 0000000000..c3d07a7628 --- /dev/null +++ b/src/content/tools/docs/2.3/index.md @@ -0,0 +1,13 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +Primarily, it describes the different [code lint tests](lint_tests/index.html) +run by `nf-core lint` (typically visited by a developer when their pipeline fails a given +test), and also reference for the `nf_core` [Python package API](api/index.html). + +# Indices and tables + +- [Index](genindex) +- [Module Index](py-modindex) +- [Search Page](search) diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.3/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..f25b5c62b4 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/files_exist.md @@ -0,0 +1,87 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yaml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.3/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..594dcbfe50 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,56 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.markdownlint.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +assets/multiqc_config.yaml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your linting config file. For example: + +```yaml +files_unchanged: + - .github/workflows/branch.yml + - assets/multiqc_config.yaml +``` + +::: diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/index.md b/src/content/tools/docs/2.3/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..d73f5fbd15 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/index.md @@ -0,0 +1,38 @@ +# Pipline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.3/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.3/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..40c566e813 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version git_sha hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.3/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.3/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.3/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.3/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.3/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.3/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.3/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.3/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.3/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.3/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.3/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.4.1/api/bump_version.md b/src/content/tools/docs/2.4.1/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.4.1/api/create.md b/src/content/tools/docs/2.4.1/api/create.md new file mode 100644 index 0000000000..1a80c6b013 --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/create.md @@ -0,0 +1,40 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.4.1/api/download.md b/src/content/tools/docs/2.4.1/api/download.md new file mode 100644 index 0000000000..2cb1eb33c3 --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/download.md @@ -0,0 +1,162 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.4.1/api/index.md b/src/content/tools/docs/2.4.1/api/index.md new file mode 100644 index 0000000000..d0b53ad8b0 --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/index.md @@ -0,0 +1,56 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.4.1/api/launch.md b/src/content/tools/docs/2.4.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.4.1/api/licences.md b/src/content/tools/docs/2.4.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.4.1/api/lint.md b/src/content/tools/docs/2.4.1/api/lint.md new file mode 100644 index 0000000000..cf1c641287 --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.4.1/api/list.md b/src/content/tools/docs/2.4.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.4.1/api/modules.md b/src/content/tools/docs/2.4.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.4.1/api/schema.md b/src/content/tools/docs/2.4.1/api/schema.md new file mode 100644 index 0000000000..76c454b25b --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/schema.md @@ -0,0 +1,138 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=['parameter', 'description', 'type,', 'default', 'required', 'hidden']){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.4.1/api/sync.md b/src/content/tools/docs/2.4.1/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.4.1/api/utils.md b/src/content/tools/docs/2.4.1/api/utils.md new file mode 100644 index 0000000000..615d5f527f --- /dev/null +++ b/src/content/tools/docs/2.4.1/api/utils.md @@ -0,0 +1,307 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.4.1/index.md b/src/content/tools/docs/2.4.1/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.4.1/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.4.1/module_lint_tests/index.md b/src/content/tools/docs/2.4.1/module_lint_tests/index.md new file mode 100644 index 0000000000..457d4b460a --- /dev/null +++ b/src/content/tools/docs/2.4.1/module_lint_tests/index.md @@ -0,0 +1,12 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) diff --git a/src/content/tools/docs/2.4.1/module_lint_tests/main_nf.md b/src/content/tools/docs/2.4.1/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..0d9a16a731 --- /dev/null +++ b/src/content/tools/docs/2.4.1/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.4.1/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.4.1/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.4.1/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.4.1/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.4.1/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.4.1/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.4.1/module_lint_tests/module_todos.md b/src/content/tools/docs/2.4.1/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.4.1/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..d95399cb7d --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,106 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..a280965365 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,55 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..5a0789075d --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/index.md @@ -0,0 +1,40 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.4.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.4.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.4.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.4/api/bump_version.md b/src/content/tools/docs/2.4/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.4/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.4/api/create.md b/src/content/tools/docs/2.4/api/create.md new file mode 100644 index 0000000000..1a80c6b013 --- /dev/null +++ b/src/content/tools/docs/2.4/api/create.md @@ -0,0 +1,40 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. diff --git a/src/content/tools/docs/2.4/api/download.md b/src/content/tools/docs/2.4/api/download.md new file mode 100644 index 0000000000..2cb1eb33c3 --- /dev/null +++ b/src/content/tools/docs/2.4/api/download.md @@ -0,0 +1,162 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `validate_md5(fname, expected=None){:python}` + +Calculates the md5sum for a file on the disk and validate with expected. + +- **Parameters:** + - **fname** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.4/api/index.md b/src/content/tools/docs/2.4/api/index.md new file mode 100644 index 0000000000..d0b53ad8b0 --- /dev/null +++ b/src/content/tools/docs/2.4/api/index.md @@ -0,0 +1,56 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.4/api/launch.md b/src/content/tools/docs/2.4/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.4/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.4/api/licences.md b/src/content/tools/docs/2.4/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.4/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.4/api/lint.md b/src/content/tools/docs/2.4/api/lint.md new file mode 100644 index 0000000000..cf1c641287 --- /dev/null +++ b/src/content/tools/docs/2.4/api/lint.md @@ -0,0 +1,138 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, md_fn=None, json_fn=None){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=(), fail_ignored=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +Solution found on Stack Overflow: + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.4/api/list.md b/src/content/tools/docs/2.4/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.4/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.4/api/modules.md b/src/content/tools/docs/2.4/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.4/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.4/api/schema.md b/src/content/tools/docs/2.4/api/schema.md new file mode 100644 index 0000000000..76c454b25b --- /dev/null +++ b/src/content/tools/docs/2.4/api/schema.md @@ -0,0 +1,138 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=['parameter', 'description', 'type,', 'default', 'required', 'hidden']){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.4/api/sync.md b/src/content/tools/docs/2.4/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.4/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.4/api/utils.md b/src/content/tools/docs/2.4/api/utils.md new file mode 100644 index 0000000000..ee2d31dc64 --- /dev/null +++ b/src/content/tools/docs/2.4/api/utils.md @@ -0,0 +1,307 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=['conda-forge', 'bioconda', 'defaults']){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.4/index.md b/src/content/tools/docs/2.4/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.4/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.4/module_lint_tests/index.md b/src/content/tools/docs/2.4/module_lint_tests/index.md new file mode 100644 index 0000000000..457d4b460a --- /dev/null +++ b/src/content/tools/docs/2.4/module_lint_tests/index.md @@ -0,0 +1,12 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) diff --git a/src/content/tools/docs/2.4/module_lint_tests/main_nf.md b/src/content/tools/docs/2.4/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..0d9a16a731 --- /dev/null +++ b/src/content/tools/docs/2.4/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.4/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.4/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.4/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.4/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.4/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.4/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.4/module_lint_tests/module_todos.md b/src/content/tools/docs/2.4/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.4/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.4/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..d95399cb7d --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/files_exist.md @@ -0,0 +1,106 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.4/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..a280965365 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,55 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/index.md b/src/content/tools/docs/2.4/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..5a0789075d --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/index.md @@ -0,0 +1,40 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.4/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.4/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.4/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.4/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.4/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.4/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.4/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.4/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.4/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..b5e29febba --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,59 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.4/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.4/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.4/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.4/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.4/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.5.1/api/bump_version.md b/src/content/tools/docs/2.5.1/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.5.1/api/create.md b/src/content/tools/docs/2.5.1/api/create.md new file mode 100644 index 0000000000..2a1b52db4a --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/create.md @@ -0,0 +1,81 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.5.1/api/download.md b/src/content/tools/docs/2.5.1/api/download.md new file mode 100644 index 0000000000..7f88029809 --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/download.md @@ -0,0 +1,152 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.5.1/api/index.md b/src/content/tools/docs/2.5.1/api/index.md new file mode 100644 index 0000000000..7e11e57698 --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/index.md @@ -0,0 +1,64 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.5.1/api/launch.md b/src/content/tools/docs/2.5.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.5.1/api/licences.md b/src/content/tools/docs/2.5.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.5.1/api/lint.md b/src/content/tools/docs/2.5.1/api/lint.md new file mode 100644 index 0000000000..0fd5b20490 --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.5.1/api/list.md b/src/content/tools/docs/2.5.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.5.1/api/modules.md b/src/content/tools/docs/2.5.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.5.1/api/schema.md b/src/content/tools/docs/2.5.1/api/schema.md new file mode 100644 index 0000000000..909af59bcf --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/schema.md @@ -0,0 +1,151 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.5.1/api/sync.md b/src/content/tools/docs/2.5.1/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.5.1/api/utils.md b/src/content/tools/docs/2.5.1/api/utils.md new file mode 100644 index 0000000000..9f4d650165 --- /dev/null +++ b/src/content/tools/docs/2.5.1/api/utils.md @@ -0,0 +1,357 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.5.1/index.md b/src/content/tools/docs/2.5.1/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.5.1/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.5.1/module_lint_tests/index.md b/src/content/tools/docs/2.5.1/module_lint_tests/index.md new file mode 100644 index 0000000000..457d4b460a --- /dev/null +++ b/src/content/tools/docs/2.5.1/module_lint_tests/index.md @@ -0,0 +1,12 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) diff --git a/src/content/tools/docs/2.5.1/module_lint_tests/main_nf.md b/src/content/tools/docs/2.5.1/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..e3011d55f0 --- /dev/null +++ b/src/content/tools/docs/2.5.1/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.5.1/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.5.1/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.5.1/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.5.1/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.5.1/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.5.1/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.5.1/module_lint_tests/module_todos.md b/src/content/tools/docs/2.5.1/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.5.1/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..eb2bf301e2 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,108 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..b355a94543 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..5a0789075d --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/index.md @@ -0,0 +1,40 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.5.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.5.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.5.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.5/api/bump_version.md b/src/content/tools/docs/2.5/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.5/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.5/api/create.md b/src/content/tools/docs/2.5/api/create.md new file mode 100644 index 0000000000..2a1b52db4a --- /dev/null +++ b/src/content/tools/docs/2.5/api/create.md @@ -0,0 +1,81 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.5/api/download.md b/src/content/tools/docs/2.5/api/download.md new file mode 100644 index 0000000000..7f88029809 --- /dev/null +++ b/src/content/tools/docs/2.5/api/download.md @@ -0,0 +1,152 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.5/api/index.md b/src/content/tools/docs/2.5/api/index.md new file mode 100644 index 0000000000..7e11e57698 --- /dev/null +++ b/src/content/tools/docs/2.5/api/index.md @@ -0,0 +1,64 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.5/api/launch.md b/src/content/tools/docs/2.5/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.5/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.5/api/licences.md b/src/content/tools/docs/2.5/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.5/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.5/api/lint.md b/src/content/tools/docs/2.5/api/lint.md new file mode 100644 index 0000000000..0fd5b20490 --- /dev/null +++ b/src/content/tools/docs/2.5/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.5/api/list.md b/src/content/tools/docs/2.5/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.5/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.5/api/modules.md b/src/content/tools/docs/2.5/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.5/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.5/api/schema.md b/src/content/tools/docs/2.5/api/schema.md new file mode 100644 index 0000000000..909af59bcf --- /dev/null +++ b/src/content/tools/docs/2.5/api/schema.md @@ -0,0 +1,151 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.5/api/sync.md b/src/content/tools/docs/2.5/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.5/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.5/api/utils.md b/src/content/tools/docs/2.5/api/utils.md new file mode 100644 index 0000000000..9f4d650165 --- /dev/null +++ b/src/content/tools/docs/2.5/api/utils.md @@ -0,0 +1,357 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.5/index.md b/src/content/tools/docs/2.5/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.5/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.5/module_lint_tests/index.md b/src/content/tools/docs/2.5/module_lint_tests/index.md new file mode 100644 index 0000000000..457d4b460a --- /dev/null +++ b/src/content/tools/docs/2.5/module_lint_tests/index.md @@ -0,0 +1,12 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) diff --git a/src/content/tools/docs/2.5/module_lint_tests/main_nf.md b/src/content/tools/docs/2.5/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..e3011d55f0 --- /dev/null +++ b/src/content/tools/docs/2.5/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.5/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.5/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.5/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.5/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.5/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.5/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.5/module_lint_tests/module_todos.md b/src/content/tools/docs/2.5/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.5/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.5/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..9c5c84a111 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/files_exist.md @@ -0,0 +1,107 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +``` + +Files that _must not_ be present: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.5/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..e59fcfcc40 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,56 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/index.md b/src/content/tools/docs/2.5/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..5a0789075d --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/index.md @@ -0,0 +1,40 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.5/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.5/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.5/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.5/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.5/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.5/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.5/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.5/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.5/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.5/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.5/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.5/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.5/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.5/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.6/api/bump_version.md b/src/content/tools/docs/2.6/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.6/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.6/api/create.md b/src/content/tools/docs/2.6/api/create.md new file mode 100644 index 0000000000..2a1b52db4a --- /dev/null +++ b/src/content/tools/docs/2.6/api/create.md @@ -0,0 +1,81 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.6/api/download.md b/src/content/tools/docs/2.6/api/download.md new file mode 100644 index 0000000000..7f88029809 --- /dev/null +++ b/src/content/tools/docs/2.6/api/download.md @@ -0,0 +1,152 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.6/api/index.md b/src/content/tools/docs/2.6/api/index.md new file mode 100644 index 0000000000..7e11e57698 --- /dev/null +++ b/src/content/tools/docs/2.6/api/index.md @@ -0,0 +1,64 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.6/api/launch.md b/src/content/tools/docs/2.6/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.6/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.6/api/licences.md b/src/content/tools/docs/2.6/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.6/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.6/api/lint.md b/src/content/tools/docs/2.6/api/lint.md new file mode 100644 index 0000000000..0fd5b20490 --- /dev/null +++ b/src/content/tools/docs/2.6/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.6/api/list.md b/src/content/tools/docs/2.6/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.6/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.6/api/modules.md b/src/content/tools/docs/2.6/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.6/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.6/api/schema.md b/src/content/tools/docs/2.6/api/schema.md new file mode 100644 index 0000000000..909af59bcf --- /dev/null +++ b/src/content/tools/docs/2.6/api/schema.md @@ -0,0 +1,151 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.6/api/sync.md b/src/content/tools/docs/2.6/api/sync.md new file mode 100644 index 0000000000..e327553c3f --- /dev/null +++ b/src/content/tools/docs/2.6/api/sync.md @@ -0,0 +1,151 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.6/api/utils.md b/src/content/tools/docs/2.6/api/utils.md new file mode 100644 index 0000000000..9f4d650165 --- /dev/null +++ b/src/content/tools/docs/2.6/api/utils.md @@ -0,0 +1,357 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(dir='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, poll_every=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **poll_every** (_int_) – How many tenths of a second to wait between function calls. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.6/index.md b/src/content/tools/docs/2.6/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.6/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.6/module_lint_tests/index.md b/src/content/tools/docs/2.6/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.6/module_lint_tests/main_nf.md b/src/content/tools/docs/2.6/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..e3011d55f0 --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.6/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.6/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.6/module_lint_tests/module_changes.md b/src/content/tools/docs/2.6/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.6/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.6/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.6/module_lint_tests/module_patch.md b/src/content/tools/docs/2.6/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..2e4bb7ea5e --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreModule){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.6/module_lint_tests/module_tests.md b/src/content/tools/docs/2.6/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..a2d227becf --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/module_tests.md @@ -0,0 +1,10 @@ +# module_tests + +#### `ModuleLint.module_tests(module){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the module is present in the `pytest_modules.yml` +file. diff --git a/src/content/tools/docs/2.6/module_lint_tests/module_todos.md b/src/content/tools/docs/2.6/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.6/module_lint_tests/module_version.md b/src/content/tools/docs/2.6/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.6/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..34f2096391 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,63 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: + +- The Docker container for the pipeline must use the correct pipeline version number: + + > - Development pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/:dev + > ``` + > - Released pipelines: + > ```bash + > docker tag nfcore/:dev nfcore/: + > ``` + > - Complete example for a released pipeline called _nf-core/example_ with version number `1.0.0`: + > ```yaml + > - name: Build new docker image + > if: env.GIT_DIFF + > run: docker build --no-cache . -t nfcore/example:1.0.0 + > ``` + + > - name: Pull docker image + > if: ${{ !env.GIT\_DIFF }} + > run: | + > docker pull nfcore/example:dev + > docker tag nfcore/example:dev nfcore/example:1.0.0 + > + > ``` + > + > ``` diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.6/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b1a3a54588 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.6/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..b355a94543 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/index.md b/src/content/tools/docs/2.6/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..3c5e3d739c --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/index.md @@ -0,0 +1,42 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.6/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.6/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.6/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.6/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.6/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.6/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.6/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.6/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.6/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.6/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.6/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.6/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.6/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.6/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.6/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.7.1/api/bump_version.md b/src/content/tools/docs/2.7.1/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.7.1/api/create.md b/src/content/tools/docs/2.7.1/api/create.md new file mode 100644 index 0000000000..2082d9a4a8 --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.7.1/api/download.md b/src/content/tools/docs/2.7.1/api/download.md new file mode 100644 index 0000000000..7f88029809 --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/download.md @@ -0,0 +1,152 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.7.1/api/index.md b/src/content/tools/docs/2.7.1/api/index.md new file mode 100644 index 0000000000..87128bfa5c --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/index.md @@ -0,0 +1,67 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.7.1/api/launch.md b/src/content/tools/docs/2.7.1/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.7.1/api/licences.md b/src/content/tools/docs/2.7.1/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.7.1/api/lint.md b/src/content/tools/docs/2.7.1/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.7.1/api/list.md b/src/content/tools/docs/2.7.1/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.7.1/api/modules.md b/src/content/tools/docs/2.7.1/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.7.1/api/schema.md b/src/content/tools/docs/2.7.1/api/schema.md new file mode 100644 index 0000000000..909af59bcf --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/schema.md @@ -0,0 +1,151 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.7.1/api/subworkflows.md b/src/content/tools/docs/2.7.1/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.7.1/api/sync.md b/src/content/tools/docs/2.7.1/api/sync.md new file mode 100644 index 0000000000..3a964e6de6 --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/sync.md @@ -0,0 +1,158 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `template_yaml{:python}` + +Path to template.yml file for pipeline creation settings. + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.7.1/api/utils.md b/src/content/tools/docs/2.7.1/api/utils.md new file mode 100644 index 0000000000..a205cafae5 --- /dev/null +++ b/src/content/tools/docs/2.7.1/api/utils.md @@ -0,0 +1,361 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.7.1/index.md b/src/content/tools/docs/2.7.1/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.7.1/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/index.md b/src/content/tools/docs/2.7.1/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/main_nf.md b/src/content/tools/docs/2.7.1/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..e3011d55f0 --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.7.1/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/module_changes.md b/src/content/tools/docs/2.7.1/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.7.1/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/module_patch.md b/src/content/tools/docs/2.7.1/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..2e4bb7ea5e --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreModule){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/module_tests.md b/src/content/tools/docs/2.7.1/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..a2d227becf --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/module_tests.md @@ -0,0 +1,10 @@ +# module_tests + +#### `ModuleLint.module_tests(module){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the module is present in the `pytest_modules.yml` +file. diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/module_todos.md b/src/content/tools/docs/2.7.1/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.7.1/module_lint_tests/module_version.md b/src/content/tools/docs/2.7.1/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.7.1/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..41e0a6fbef --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b1a3a54588 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..b355a94543 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/index.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..3c5e3d739c --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/index.md @@ -0,0 +1,42 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.7.1/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.7.1/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.7.1/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.7.2/api/bump_version.md b/src/content/tools/docs/2.7.2/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.7.2/api/create.md b/src/content/tools/docs/2.7.2/api/create.md new file mode 100644 index 0000000000..2082d9a4a8 --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.7.2/api/download.md b/src/content/tools/docs/2.7.2/api/download.md new file mode 100644 index 0000000000..7f88029809 --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/download.md @@ -0,0 +1,152 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.7.2/api/index.md b/src/content/tools/docs/2.7.2/api/index.md new file mode 100644 index 0000000000..87128bfa5c --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/index.md @@ -0,0 +1,67 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.7.2/api/launch.md b/src/content/tools/docs/2.7.2/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.7.2/api/licences.md b/src/content/tools/docs/2.7.2/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.7.2/api/lint.md b/src/content/tools/docs/2.7.2/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.7.2/api/list.md b/src/content/tools/docs/2.7.2/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.7.2/api/modules.md b/src/content/tools/docs/2.7.2/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.7.2/api/schema.md b/src/content/tools/docs/2.7.2/api/schema.md new file mode 100644 index 0000000000..909af59bcf --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/schema.md @@ -0,0 +1,151 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.7.2/api/subworkflows.md b/src/content/tools/docs/2.7.2/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.7.2/api/sync.md b/src/content/tools/docs/2.7.2/api/sync.md new file mode 100644 index 0000000000..3a964e6de6 --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/sync.md @@ -0,0 +1,158 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `template_yaml{:python}` + +Path to template.yml file for pipeline creation settings. + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.7.2/api/utils.md b/src/content/tools/docs/2.7.2/api/utils.md new file mode 100644 index 0000000000..a205cafae5 --- /dev/null +++ b/src/content/tools/docs/2.7.2/api/utils.md @@ -0,0 +1,361 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.7.2/index.md b/src/content/tools/docs/2.7.2/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.7.2/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/index.md b/src/content/tools/docs/2.7.2/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/main_nf.md b/src/content/tools/docs/2.7.2/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..e3011d55f0 --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.7.2/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/module_changes.md b/src/content/tools/docs/2.7.2/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.7.2/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/module_patch.md b/src/content/tools/docs/2.7.2/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..2e4bb7ea5e --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreModule){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/module_tests.md b/src/content/tools/docs/2.7.2/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..a2d227becf --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/module_tests.md @@ -0,0 +1,10 @@ +# module_tests + +#### `ModuleLint.module_tests(module){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the module is present in the `pytest_modules.yml` +file. diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/module_todos.md b/src/content/tools/docs/2.7.2/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.7.2/module_lint_tests/module_version.md b/src/content/tools/docs/2.7.2/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.7.2/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..41e0a6fbef --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b1a3a54588 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..b355a94543 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/index.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..3c5e3d739c --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/index.md @@ -0,0 +1,42 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.7.2/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.7.2/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.7.2/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.7/api/bump_version.md b/src/content/tools/docs/2.7/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.7/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.7/api/create.md b/src/content/tools/docs/2.7/api/create.md new file mode 100644 index 0000000000..2082d9a4a8 --- /dev/null +++ b/src/content/tools/docs/2.7/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.7/api/download.md b/src/content/tools/docs/2.7/api/download.md new file mode 100644 index 0000000000..7f88029809 --- /dev/null +++ b/src/content/tools/docs/2.7/api/download.md @@ -0,0 +1,152 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘quay.io/biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.7/api/index.md b/src/content/tools/docs/2.7/api/index.md new file mode 100644 index 0000000000..87128bfa5c --- /dev/null +++ b/src/content/tools/docs/2.7/api/index.md @@ -0,0 +1,67 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.7/api/launch.md b/src/content/tools/docs/2.7/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.7/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.7/api/licences.md b/src/content/tools/docs/2.7/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.7/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.7/api/lint.md b/src/content/tools/docs/2.7/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.7/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.7/api/list.md b/src/content/tools/docs/2.7/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.7/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.7/api/modules.md b/src/content/tools/docs/2.7/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.7/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.7/api/schema.md b/src/content/tools/docs/2.7/api/schema.md new file mode 100644 index 0000000000..909af59bcf --- /dev/null +++ b/src/content/tools/docs/2.7/api/schema.md @@ -0,0 +1,151 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.7/api/subworkflows.md b/src/content/tools/docs/2.7/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.7/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.7/api/sync.md b/src/content/tools/docs/2.7/api/sync.md new file mode 100644 index 0000000000..3a964e6de6 --- /dev/null +++ b/src/content/tools/docs/2.7/api/sync.md @@ -0,0 +1,158 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `template_yaml{:python}` + +Path to template.yml file for pipeline creation settings. + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.7/api/utils.md b/src/content/tools/docs/2.7/api/utils.md new file mode 100644 index 0000000000..a205cafae5 --- /dev/null +++ b/src/content/tools/docs/2.7/api/utils.md @@ -0,0 +1,361 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.7/index.md b/src/content/tools/docs/2.7/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.7/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.7/module_lint_tests/index.md b/src/content/tools/docs/2.7/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.7/module_lint_tests/main_nf.md b/src/content/tools/docs/2.7/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..e3011d55f0 --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.7/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.7/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..594e745e84 --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it contains +the required keys: `name`, input\`\` and +`output`. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.7/module_lint_tests/module_changes.md b/src/content/tools/docs/2.7/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.7/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.7/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.7/module_lint_tests/module_patch.md b/src/content/tools/docs/2.7/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..2e4bb7ea5e --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreModule){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.7/module_lint_tests/module_tests.md b/src/content/tools/docs/2.7/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..a2d227becf --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/module_tests.md @@ -0,0 +1,10 @@ +# module_tests + +#### `ModuleLint.module_tests(module){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the module is present in the `pytest_modules.yml` +file. diff --git a/src/content/tools/docs/2.7/module_lint_tests/module_todos.md b/src/content/tools/docs/2.7/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.7/module_lint_tests/module_version.md b/src/content/tools/docs/2.7/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.7/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..41e0a6fbef --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.7/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b1a3a54588 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.7/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..b355a94543 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/index.md b/src/content/tools/docs/2.7/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..3c5e3d739c --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/index.md @@ -0,0 +1,42 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.7/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.7/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.7/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.7/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.7/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..1e04ccd548 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,92 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core-lint.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +nextflow_config: False +``` diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.7/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.7/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.7/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..51b8040c97 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/readme.md @@ -0,0 +1,25 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` +- Bioconda badge + - If your pipeline contains a file called `environment.yml` in the root directory, a bioconda badge is required + - Required badge code: + ```md + [![install with bioconda](https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg)](https://bioconda.github.io/) + ``` + +:::note +These badges are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.7/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.7/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.7/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.7/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.7/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.7/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.7/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.8/api/bump_version.md b/src/content/tools/docs/2.8/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.8/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.8/api/create.md b/src/content/tools/docs/2.8/api/create.md new file mode 100644 index 0000000000..2082d9a4a8 --- /dev/null +++ b/src/content/tools/docs/2.8/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.8/api/download.md b/src/content/tools/docs/2.8/api/download.md new file mode 100644 index 0000000000..62b9f02425 --- /dev/null +++ b/src/content/tools/docs/2.8/api/download.md @@ -0,0 +1,152 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, container=None, singularity_cache_only=False, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_str_) – The workflow revision to download, like 1.0. Defaults to None. + - **singularity** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `find_container_images(){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic. + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw source code that will likely need to be cleaned. + +If multiple containers are found, prioritise any prefixed with http for direct download. + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘biocontainers/fastqc:0.11.9–0’ }” + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch + +#### `prompt_singularity_cachedir_only(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `prompt_use_singularity_cachedir(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(){:python}` + +Edit the downloaded nextflow.config file to use the local config files diff --git a/src/content/tools/docs/2.8/api/index.md b/src/content/tools/docs/2.8/api/index.md new file mode 100644 index 0000000000..87128bfa5c --- /dev/null +++ b/src/content/tools/docs/2.8/api/index.md @@ -0,0 +1,67 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.8/api/launch.md b/src/content/tools/docs/2.8/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.8/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.8/api/licences.md b/src/content/tools/docs/2.8/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.8/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.8/api/lint.md b/src/content/tools/docs/2.8/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.8/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.8/api/list.md b/src/content/tools/docs/2.8/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.8/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.8/api/modules.md b/src/content/tools/docs/2.8/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.8/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.8/api/schema.md b/src/content/tools/docs/2.8/api/schema.md new file mode 100644 index 0000000000..d795ff7452 --- /dev/null +++ b/src/content/tools/docs/2.8/api/schema.md @@ -0,0 +1,165 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.8/api/subworkflows.md b/src/content/tools/docs/2.8/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.8/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.8/api/sync.md b/src/content/tools/docs/2.8/api/sync.md new file mode 100644 index 0000000000..3a964e6de6 --- /dev/null +++ b/src/content/tools/docs/2.8/api/sync.md @@ -0,0 +1,158 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `template_yaml{:python}` + +Path to template.yml file for pipeline creation settings. + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.8/api/utils.md b/src/content/tools/docs/2.8/api/utils.md new file mode 100644 index 0000000000..a205cafae5 --- /dev/null +++ b/src/content/tools/docs/2.8/api/utils.md @@ -0,0 +1,361 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.8/index.md b/src/content/tools/docs/2.8/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.8/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.8/module_lint_tests/index.md b/src/content/tools/docs/2.8/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.8/module_lint_tests/main_nf.md b/src/content/tools/docs/2.8/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..e3011d55f0 --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.8/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.8/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..b1045c5fbd --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.8/module_lint_tests/module_changes.md b/src/content/tools/docs/2.8/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.8/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.8/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.8/module_lint_tests/module_patch.md b/src/content/tools/docs/2.8/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..2e4bb7ea5e --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreModule){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.8/module_lint_tests/module_tests.md b/src/content/tools/docs/2.8/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..a2d227becf --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/module_tests.md @@ -0,0 +1,10 @@ +# module_tests + +#### `ModuleLint.module_tests(module){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the module is present in the `pytest_modules.yml` +file. diff --git a/src/content/tools/docs/2.8/module_lint_tests/module_todos.md b/src/content/tools/docs/2.8/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.8/module_lint_tests/module_version.md b/src/content/tools/docs/2.8/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.8/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..41e0a6fbef --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.8/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b1a3a54588 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/files_exist.md @@ -0,0 +1,111 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.8/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..b355a94543 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,57 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreSchema.groovy +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/index.md b/src/content/tools/docs/2.8/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..c8cf0fe7b1 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/index.md @@ -0,0 +1,44 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [system_exit](system_exit) + - [`PipelineLint.system_exit()`](system_exit#nf_core.lint.PipelineLint.system_exit) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.8/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.8/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.8/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.8/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.8/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..035468b50e --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,94 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.show_hidden_params` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.schema_ignore_params` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.8/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.8/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.8/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.8/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.8/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.8/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/2.8/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.8/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.8/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.8/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.8/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/2.9/api/bump_version.md b/src/content/tools/docs/2.9/api/bump_version.md new file mode 100644 index 0000000000..3f45f9fefb --- /dev/null +++ b/src/content/tools/docs/2.9/api/bump_version.md @@ -0,0 +1,35 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj, new_version){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj, new_version){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename, pipeline_obj, patterns){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. + - **newstr** (_str_) – The replaced string. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/2.9/api/create.md b/src/content/tools/docs/2.9/api/create.md new file mode 100644 index 0000000000..2082d9a4a8 --- /dev/null +++ b/src/content/tools/docs/2.9/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + +#### `download_pipeline_logo(url, img_fn){:python}` + +Attempt to download a logo from the website. Retry if it fails. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/2.9/api/download.md b/src/content/tools/docs/2.9/api/download.md new file mode 100644 index 0000000000..4efb61fc6e --- /dev/null +++ b/src/content/tools/docs/2.9/api/download.md @@ -0,0 +1,286 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`exception{:python}`_`nf_core.download.ContainerError(container, registry, address, out_path, singularity_command, error_msg){:python}` + +Bases: `Exception` + +A class of errors related to pulling containers with Singularity/Apptainer + +#### `undefined{:python}` ImageExists(error_log) + +Bases: `FileExistsError` + +Image already exists in cache/output directory. + +#### `undefined{:python}` ImageNotFound(error_log) + +Bases: `FileNotFoundError` + +The image can not be found in the registry + +#### `undefined{:python}` InvalidTag(error_log) + +Bases: `AttributeError` + +Image and registry are valid, but the (version) tag is not + +#### `undefined{:python}` OtherError(error_log) + +Bases: `RuntimeError` + +Undefined error with the container + +#### `undefined{:python}` RegistryNotFound(error_log) + +Bases: `ConnectionRefusedError` + +The specified registry does not resolve to a valid IP address + +### _`exception{:python}`_`nf_core.download.DownloadError{:python}` + +Bases: `RuntimeError` + +A custom exception that is raised when nf-core download encounters a problem that we already took into consideration. +In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, tower=False, download_configuration=None, container_system=None, container_library=None, container_cache_utilisation=None, container_cache_index=None, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_List_\*\[**str**]\*) – The workflow revision to download, like 1.0. Defaults to None. + - **container** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **tower** (_bool_) – Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(revision, wf_sha, download_url){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `download_workflow_static(){:python}` + +Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner. + +#### `download_workflow_tower(location=None){:python}` + +Create a bare-cloned git repository of the workflow, so it can be launched with tw launch as pipeline + +#### `find_container_images(workflow_directory){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic, +but not always, e.g. not for differentialabundance 1.2.0 + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw matches that will likely need to be cleaned. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(current_revision=''){:python}` + +Loop through container names and download Singularity images + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_config_inclusion(){:python}` + +Prompt for inclusion of institutional configurations + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch +Prompt user for revision tag if ‘–revision’ was not set +If –tower is specified, allow to select multiple revisions +Also the static download allows for multiple revisions, but +we do not prompt this option interactively. + +#### `prompt_singularity_cachedir_creation(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `prompt_singularity_cachedir_remote(){:python}` + +Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR + +#### `prompt_singularity_cachedir_utilization(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `read_remote_containers(){:python}` + +Reads the file specified as index for the remote Singularity cache dir + +#### `rectify_raw_container_matches(raw_findings){:python}` + +Helper function to rectify the raw extracted container matches into fully qualified container names. +If multiple containers are found, any prefixed with http for direct download is prioritized + +Example syntax: + +Early DSL2: +: if (workflow.containerEngine == ‘singularity’ && !params.singularity_pull_docker_container) { +: container “[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)”
      +} else { +: container “quay.io/biocontainers/fastqc:0.11.9–0”
      +} + +Later DSL2: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity_pull_docker_container ? +: ‘[https://depot.galaxyproject.org/singularity/fastqc:0.11.9–0](https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0)’ : +‘biocontainers/fastqc:0.11.9–0’ }” + +Later DSL2, variable is being used: +: container “${ workflow.containerEngine == ‘singularity’ && !task.ext.singularity\_pull\_docker\_container ? +: “/${container_id}” : +“quay.io/biocontainers/${container_id}” }”
      +container_id = ‘mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0’ + +DSL1 / Special case DSL2: +: container “nfcore/cellranger:6.0.2” + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, library, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. + - **library** (_list_ _of_ _str_) – A list of libraries to try for pulling the image. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(revision_dirname){:python}` + +Edit the downloaded nextflow.config file to use the local config files + +### _`class{:python}`_`nf_core.download.WorkflowRepo(remote_url, revision, commit, location=None, hide_progress=False, in_cache=True){:python}` + +Bases: `SyncedRepo` + +An object to store details about a locally cached workflow repository. + +Important Attributes: +: fullname: The full name of the repository, `nf-core/{self.pipelinename}`. +local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to `$HOME/.cache/nf-core/nf-core/{self.pipeline}`. + +#### `access(){:python}` + +#### `bare_clone(destination){:python}` + +#### `checkout(commit){:python}` + +Checks out the repository at the requested commit + +- **Parameters:** + **commit** (_str_) – Git SHA of the commit + +#### `get_remote_branches(remote_url){:python}` + +Get all branches from a remote repository + +- **Parameters:** + **remote_url** (_str_) – The git url to the remote repository +- **Returns:** + All branches found in the remote +- **Return type:** + (set\[str]) + +#### `retry_setup_local_repo(skip_confirm=False){:python}` + +#### `setup_local_repo(remote, location=None, in_cache=True){:python}` + +Sets up the local git repository. If the repository has been cloned previously, it +returns a git.Repo object of that clone. Otherwise it tries to clone the repository from +the provided remote URL and returns a git.Repo of the new clone. + +- **Parameters:** + - **remote** (_str_) – git url of remote + - **location** (_Path_) – location where the clone should be created/cached. + - **in_cache** (_bool_\*,\* _optional_) – Whether to clone the repository from the cache. Defaults to False. + +Sets self.repo + +#### `tidy_tags_and_branches(){:python}` + +Function to delete all tags and branches that are not of interest to the downloader. +This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + +However, due to local caching, the downloader might also want access to revisions that had been deleted before. +In that case, don’t bother with re-adding the tags and rather download anew from Github. diff --git a/src/content/tools/docs/2.9/api/index.md b/src/content/tools/docs/2.9/api/index.md new file mode 100644 index 0000000000..58029636f5 --- /dev/null +++ b/src/content/tools/docs/2.9/api/index.md @@ -0,0 +1,72 @@ +# API Reference + +# Tests: + +- [nf_core.bump_version](bump_version) + - [`bump_nextflow_version()`](bump_version#nf_core.bump_version.bump_nextflow_version) + - [`bump_pipeline_version()`](bump_version#nf_core.bump_version.bump_pipeline_version) + - [`update_file_version()`](bump_version#nf_core.bump_version.update_file_version) +- [nf_core.create](create) + - [`PipelineCreate`](create#nf_core.create.PipelineCreate) +- [nf_core.download](download) + - [`ContainerError`](download#nf_core.download.ContainerError) + - [`DownloadError`](download#nf_core.download.DownloadError) + - [`DownloadProgress`](download#nf_core.download.DownloadProgress) + - [`DownloadWorkflow`](download#nf_core.download.DownloadWorkflow) + - [`WorkflowRepo`](download#nf_core.download.WorkflowRepo) +- [nf_core.launch](launch) + - [`Launch`](launch#nf_core.launch.Launch) +- [nf_core.licences](licences) + - [`WorkflowLicences`](licences#nf_core.licences.WorkflowLicences) +- [nf_core.lint](lint) + - [`run_linting()`](lint#nf_core.lint.run_linting) + - [`PipelineLint`](lint#nf_core.lint.PipelineLint) +- [nf_core.list](list) + - [`LocalWorkflow`](list#nf_core.list.LocalWorkflow) + - [`RemoteWorkflow`](list#nf_core.list.RemoteWorkflow) + - [`Workflows`](list#nf_core.list.Workflows) + - [`get_local_wf()`](list#nf_core.list.get_local_wf) + - [`list_workflows()`](list#nf_core.list.list_workflows) + - [`pretty_date()`](list#nf_core.list.pretty_date) +- [nf_core.modules](modules) +- [nf_core.schema](schema) + - [`PipelineSchema`](schema#nf_core.schema.PipelineSchema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) + - [`PipelineSync`](sync#nf_core.sync.PipelineSync) + - [`PullRequestException`](sync#nf_core.sync.PullRequestException) + - [`SyncException`](sync#nf_core.sync.SyncException) +- [nf_core.utils](utils) + - [`GitHub_API_Session`](utils#nf_core.utils.GitHub_API_Session) + - [`Pipeline`](utils#nf_core.utils.Pipeline) + - [`SingularityCacheFilePathValidator`](utils#nf_core.utils.SingularityCacheFilePathValidator) + - [`anaconda_package()`](utils#nf_core.utils.anaconda_package) + - [`check_if_outdated()`](utils#nf_core.utils.check_if_outdated) + - [`custom_yaml_dumper()`](utils#nf_core.utils.custom_yaml_dumper) + - [`determine_base_dir()`](utils#nf_core.utils.determine_base_dir) + - [`fetch_remote_version()`](utils#nf_core.utils.fetch_remote_version) + - [`fetch_wf_config()`](utils#nf_core.utils.fetch_wf_config) + - [`file_md5()`](utils#nf_core.utils.file_md5) + - [`get_biocontainer_tag()`](utils#nf_core.utils.get_biocontainer_tag) + - [`get_first_available_path()`](utils#nf_core.utils.get_first_available_path) + - [`get_repo_releases_branches()`](utils#nf_core.utils.get_repo_releases_branches) + - [`is_file_binary()`](utils#nf_core.utils.is_file_binary) + - [`is_pipeline_directory()`](utils#nf_core.utils.is_pipeline_directory) + - [`is_relative_to()`](utils#nf_core.utils.is_relative_to) + - [`load_tools_config()`](utils#nf_core.utils.load_tools_config) + - [`nextflow_cmd()`](utils#nf_core.utils.nextflow_cmd) + - [`parse_anaconda_licence()`](utils#nf_core.utils.parse_anaconda_licence) + - [`pip_package()`](utils#nf_core.utils.pip_package) + - [`plural_es()`](utils#nf_core.utils.plural_es) + - [`plural_s()`](utils#nf_core.utils.plural_s) + - [`plural_y()`](utils#nf_core.utils.plural_y) + - [`poll_nfcore_web_api()`](utils#nf_core.utils.poll_nfcore_web_api) + - [`prompt_pipeline_release_branch()`](utils#nf_core.utils.prompt_pipeline_release_branch) + - [`prompt_remote_pipeline_name()`](utils#nf_core.utils.prompt_remote_pipeline_name) + - [`rich_force_colors()`](utils#nf_core.utils.rich_force_colors) + - [`setup_nfcore_dir()`](utils#nf_core.utils.setup_nfcore_dir) + - [`setup_requests_cachedir()`](utils#nf_core.utils.setup_requests_cachedir) + - [`sort_dictionary()`](utils#nf_core.utils.sort_dictionary) + - [`strip_ansi_codes()`](utils#nf_core.utils.strip_ansi_codes) + - [`validate_file_md5()`](utils#nf_core.utils.validate_file_md5) + - [`wait_cli_function()`](utils#nf_core.utils.wait_cli_function) diff --git a/src/content/tools/docs/2.9/api/launch.md b/src/content/tools/docs/2.9/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/2.9/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/2.9/api/licences.md b/src/content/tools/docs/2.9/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/2.9/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/2.9/api/lint.md b/src/content/tools/docs/2.9/api/lint.md new file mode 100644 index 0000000000..77dffd9180 --- /dev/null +++ b/src/content/tools/docs/2.9/api/lint.md @@ -0,0 +1,132 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/2.9/api/list.md b/src/content/tools/docs/2.9/api/list.md new file mode 100644 index 0000000000..2fc6448de0 --- /dev/null +++ b/src/content/tools/docs/2.9/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/2.9/api/modules.md b/src/content/tools/docs/2.9/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/2.9/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/2.9/api/schema.md b/src/content/tools/docs/2.9/api/schema.md new file mode 100644 index 0000000000..d795ff7452 --- /dev/null +++ b/src/content/tools/docs/2.9/api/schema.md @@ -0,0 +1,165 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/2.9/api/subworkflows.md b/src/content/tools/docs/2.9/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/2.9/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/2.9/api/sync.md b/src/content/tools/docs/2.9/api/sync.md new file mode 100644 index 0000000000..3a964e6de6 --- /dev/null +++ b/src/content/tools/docs/2.9/api/sync.md @@ -0,0 +1,158 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `template_yaml{:python}` + +Path to template.yml file for pipeline creation settings. + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestException{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncException{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/2.9/api/utils.md b/src/content/tools/docs/2.9/api/utils.md new file mode 100644 index 0000000000..ad00c3a5a8 --- /dev/null +++ b/src/content/tools/docs/2.9/api/utils.md @@ -0,0 +1,380 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHub_API_Session{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### _`class{:python}`_`nf_core.utils.SingularityCacheFilePathValidator{:python}` + +Bases: `Validator` + +Validator for file path specified as –singularity-cache-index argument in nf-core download + +#### `_abc_impl{:python}`_= <\_abc.\_abc_data object>_ + +#### `validate(value){:python}` + +Validate the input. +If invalid, this should raise a `ValidationError`. + +- **Parameters:** + **document** – `Document` instance. + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_remote_version(source_url){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory='.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nextflow_cmd(cmd){:python}` + +Run a Nextflow command and capture the output. Handle errors nicely + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API + - **multiple** (_bool_) – Allow selection of multiple releases & branches (for Tower) +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/2.9/index.md b/src/content/tools/docs/2.9/index.md new file mode 100644 index 0000000000..219ae2a0ac --- /dev/null +++ b/src/content/tools/docs/2.9/index.md @@ -0,0 +1,11 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [nf-core/tools Python package API reference](api/index) +- [Index](genindex) +- [Module Index](py-modindex) diff --git a/src/content/tools/docs/2.9/module_lint_tests/index.md b/src/content/tools/docs/2.9/module_lint_tests/index.md new file mode 100644 index 0000000000..9a9ad64e61 --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/index.md @@ -0,0 +1,20 @@ +# Module lint tests + +# Tests: + +- [main_nf](main_nf) + - [`ModuleLint.main_nf()`](main_nf#nf_core.modules.lint.ModuleLint.main_nf) +- [meta_yml](meta_yml) + - [`ModuleLint.meta_yml()`](meta_yml#nf_core.modules.lint.ModuleLint.meta_yml) +- [module_changes](module_changes) + - [`ModuleLint.module_changes()`](module_changes#nf_core.modules.lint.ModuleLint.module_changes) +- [module_deprecations](module_deprecations) + - [`ModuleLint.module_deprecations()`](module_deprecations#nf_core.modules.lint.ModuleLint.module_deprecations) +- [module_patch](module_patch) + - [`ModuleLint.module_patch()`](module_patch#nf_core.modules.lint.ModuleLint.module_patch) +- [module_tests](module_tests) + - [`ModuleLint.module_tests()`](module_tests#nf_core.modules.lint.ModuleLint.module_tests) +- [module_todos](module_todos) + - [`ModuleLint.module_todos()`](module_todos#nf_core.modules.lint.ModuleLint.module_todos) +- [module_version](module_version) + - [`ModuleLint.module_version()`](module_version#nf_core.modules.lint.ModuleLint.module_version) diff --git a/src/content/tools/docs/2.9/module_lint_tests/main_nf.md b/src/content/tools/docs/2.9/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..aa42cef6d7 --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, registry, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/2.9/module_lint_tests/meta_yml.md b/src/content/tools/docs/2.9/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..b1045c5fbd --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/meta_yml.md @@ -0,0 +1,14 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. diff --git a/src/content/tools/docs/2.9/module_lint_tests/module_changes.md b/src/content/tools/docs/2.9/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/2.9/module_lint_tests/module_deprecations.md b/src/content/tools/docs/2.9/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/2.9/module_lint_tests/module_patch.md b/src/content/tools/docs/2.9/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..2e4bb7ea5e --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreModule){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/2.9/module_lint_tests/module_tests.md b/src/content/tools/docs/2.9/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..a2d227becf --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/module_tests.md @@ -0,0 +1,10 @@ +# module_tests + +#### `ModuleLint.module_tests(module){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf` and a `test.yml`, +and that the module is present in the `pytest_modules.yml` +file. diff --git a/src/content/tools/docs/2.9/module_lint_tests/module_todos.md b/src/content/tools/docs/2.9/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.9/module_lint_tests/module_version.md b/src/content/tools/docs/2.9/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/2.9/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..41e0a6fbef --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `nxf_ver` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + nxf_ver: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `nxf_ver` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/2.9/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..e911649318 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/files_exist.md @@ -0,0 +1,110 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/2.9/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..6247c86c45 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,56 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/nfcore_external_java_deps.jar +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/index.md b/src/content/tools/docs/2.9/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..c8cf0fe7b1 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/index.md @@ -0,0 +1,44 @@ +# Pipeline lint tests + +# Tests: + +- [actions_awsfulltest](actions_awsfulltest) + - [`PipelineLint.actions_awsfulltest()`](actions_awsfulltest#nf_core.lint.PipelineLint.actions_awsfulltest) +- [actions_awstest](actions_awstest) + - [`PipelineLint.actions_awstest()`](actions_awstest#nf_core.lint.PipelineLint.actions_awstest) +- [actions_ci](actions_ci) + - [`PipelineLint.actions_ci()`](actions_ci#nf_core.lint.PipelineLint.actions_ci) +- [actions_schema_validation](actions_schema_validation) + - [`PipelineLint.actions_schema_validation()`](actions_schema_validation#nf_core.lint.PipelineLint.actions_schema_validation) +- [files_exist](files_exist) + - [`PipelineLint.files_exist()`](files_exist#nf_core.lint.PipelineLint.files_exist) +- [files_unchanged](files_unchanged) + - [`PipelineLint.files_unchanged()`](files_unchanged#nf_core.lint.PipelineLint.files_unchanged) +- [merge_markers](merge_markers) + - [`PipelineLint.merge_markers()`](merge_markers#nf_core.lint.PipelineLint.merge_markers) +- [nextflow_config](modules_json) + - [`PipelineLint.modules_json()`](modules_json#nf_core.lint.PipelineLint.modules_json) +- [modules_structure](modules_structure) + - [`PipelineLint.modules_structure()`](modules_structure#nf_core.lint.PipelineLint.modules_structure) +- [multiqc_config](multiqc_config) + - [`PipelineLint.multiqc_config()`](multiqc_config#nf_core.lint.PipelineLint.multiqc_config) +- [nextflow_config](nextflow_config) + - [`PipelineLint.nextflow_config()`](nextflow_config#nf_core.lint.PipelineLint.nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) + - [`PipelineLint.pipeline_name_conventions()`](pipeline_name_conventions#nf_core.lint.PipelineLint.pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) + - [`PipelineLint.pipeline_todos()`](pipeline_todos#nf_core.lint.PipelineLint.pipeline_todos) +- [readme](readme) + - [`PipelineLint.readme()`](readme#nf_core.lint.PipelineLint.readme) +- [schema_description](schema_description) + - [`PipelineLint.schema_description()`](schema_description#nf_core.lint.PipelineLint.schema_description) +- [schema_lint](schema_lint) + - [`PipelineLint.schema_lint()`](schema_lint#nf_core.lint.PipelineLint.schema_lint) +- [schema_params](schema_params) + - [`PipelineLint.schema_params()`](schema_params#nf_core.lint.PipelineLint.schema_params) +- [system_exit](system_exit) + - [`PipelineLint.system_exit()`](system_exit#nf_core.lint.PipelineLint.system_exit) +- [template_strings](template_strings) + - [`PipelineLint.template_strings()`](template_strings#nf_core.lint.PipelineLint.template_strings) +- [version_consistency](version_consistency) + - [`PipelineLint.version_consistency()`](version_consistency#nf_core.lint.PipelineLint.version_consistency) diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/2.9/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..38076814b1 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,8 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/2.9/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/2.9/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..bc41ede4be --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,9 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: +: modules/nf-core/TOOL/SUBTOOL + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: +: modules/nf-core/modules/TOOL/SUBTOOL diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/2.9/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..a601c77f7f --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,6 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/2.9/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..14c9d5322d --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,94 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.validationShowHiddenParams` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.validationSchemaIgnoreParams` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/2.9/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/2.9/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/readme.md b/src/content/tools/docs/2.9/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/2.9/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..fafc6d6c59 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/schema_description.md @@ -0,0 +1,11 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +Furthermore warns about parameters outside of groups + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/2.9/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/2.9/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/2.9/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/2.9/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..23c06bb63d --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/template_strings.md @@ -0,0 +1,17 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. diff --git a/src/content/tools/docs/2.9/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/2.9/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/2.9/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/dev/api/bump_version.md b/src/content/tools/docs/dev/api/bump_version.md new file mode 100644 index 0000000000..107f374a18 --- /dev/null +++ b/src/content/tools/docs/dev/api/bump_version.md @@ -0,0 +1,34 @@ +# nf_core.bump_version + +Bumps the version number in all appropriate files for +a nf-core pipeline. + +### `nf_core.bump_version.bump_nextflow_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps the required Nextflow version number of a pipeline. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the required Nextflow version. + +### `nf_core.bump_version.bump_pipeline_version(pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, new_version: str){:python}` + +Bumps a pipeline version number. + +- **Parameters:** + - **pipeline_obj** ([_nf_core.utils.Pipeline_](utils#nf_core.utils.Pipeline)) – A Pipeline object that holds information + about the pipeline contents and build files. + - **new_version** (_str_) – The new version tag for the pipeline. Semantic versioning only. + +### `nf_core.bump_version.update_file_version(filename: str | Path, pipeline_obj:{:python}`[`Pipeline{:python}`](utils#nf_core.utils.Pipeline)`, patterns: List[Tuple[str, str]]){:python}` + +Updates the version number in a requested file. + +- **Parameters:** + - **filename** (_str_) – File to scan. + - **pipeline_obj** ([_nf_core.lint.PipelineLint_](lint#nf_core.lint.PipelineLint)) – A PipelineLint object that holds information + about the pipeline contents and build files. + - **pattern** (_str_) – Regex pattern to apply. +- **Raises:** + **ValueError**\*\*,\*\* **if the version number cannot be found.** – diff --git a/src/content/tools/docs/dev/api/create.md b/src/content/tools/docs/dev/api/create.md new file mode 100644 index 0000000000..048529adb0 --- /dev/null +++ b/src/content/tools/docs/dev/api/create.md @@ -0,0 +1,84 @@ +# nf_core.create + +Creates a nf-core pipeline matching the current +organization’s specification based on a template. + +### _`class{:python}`_`nf_core.create.PipelineCreate(name, description, author, version='1.0dev', no_git=False, force=False, outdir=None, template_yaml_path=None, plain=False, default_branch=None){:python}` + +Bases: `object` + +Creates a nf-core pipeline a la carte from the nf-core best-practice template. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. Semantic versioning only. Defaults to 1.0dev. + - **no_git** (_bool_) – Prevents the creation of a local Git repository for the pipeline. Defaults to False. + - **force** (_bool_) – Overwrites a given workflow directory with the same name. Defaults to False. + May the force be with you. + - **outdir** (_str_) – Path to the local output directory. + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. + - **plain** (_bool_) – If true the Git repository will be initialized plain. + - **default_branch** (_str_) – Specifies the –initial-branch name. + +#### `create_param_dict(name, description, author, version, template_yaml_path, plain, pipeline_dir){:python}` + +Creates a dictionary of parameters for the new pipeline. + +- **Parameters:** + - **name** (_str_) – Name for the pipeline. + - **description** (_str_) – Description for the pipeline. + - **author** (_str_) – Authors name of the pipeline. + - **version** (_str_) – Version flag. + - **template_yaml_path** (_str_) – Path to YAML file containing template parameters. + - **plain** (_bool_) – If true the pipeline template will be initialized plain, without customisation. + - **pipeline_dir** (_str_) – Path to the pipeline directory. + +#### `customize_template(template_areas){:python}` + +Customizes the template parameters. + +- **Parameters:** + **template_areas** (_list_) – List of available template areas to skip. + +#### `fix_linting(){:python}` + +Updates the .nf-core.yml with linting configurations +for a customized pipeline. + +#### `get_param(param_name, passed_value, template_yaml, template_yaml_path){:python}` + +#### `git_init_pipeline(){:python}` + +Initialises the new pipeline as a Git repository and submits first commit. + +- **Raises:** + **UserWarning** – if Git default branch is set to ‘dev’ or ‘TEMPLATE’. + +#### `init_pipeline(){:python}` + +Creates the nf-core pipeline. + +#### `make_pipeline_logo(){:python}` + +Fetch a logo for the new pipeline from the nf-core website + +#### `prompt_wf_author(){:python}` + +#### `prompt_wf_description(){:python}` + +#### `prompt_wf_name(){:python}` + +#### `remove_nf_core_in_bug_report_template(){:python}` + +Remove the field mentioning nf-core documentation +in the github bug report template + +#### `render_template(){:python}` + +Runs Jinja to create a new nf-core pipeline. + +#### `update_nextflow_schema(){:python}` + +Removes unused parameters from the nextflow schema. diff --git a/src/content/tools/docs/dev/api/download.md b/src/content/tools/docs/dev/api/download.md new file mode 100644 index 0000000000..35c7706bfc --- /dev/null +++ b/src/content/tools/docs/dev/api/download.md @@ -0,0 +1,322 @@ +# nf_core.download + +Downloads a nf-core pipeline to the local file system. + +### _`exception{:python}`_`nf_core.download.ContainerError(container, registry, address, absolute_URI, out_path, singularity_command, error_msg){:python}` + +Bases: `Exception` + +A class of errors related to pulling containers with Singularity/Apptainer + +#### _`exception{:python}`_`ImageExistsError(error_log){:python}` + +Bases: `FileExistsError` + +Image already exists in cache/output directory. + +#### _`exception{:python}`_`ImageNotFoundError(error_log){:python}` + +Bases: `FileNotFoundError` + +The image can not be found in the registry + +#### _`exception{:python}`_`InvalidTagError(error_log){:python}` + +Bases: `AttributeError` + +Image and registry are valid, but the (version) tag is not + +#### _`exception{:python}`_`OtherError(error_log){:python}` + +Bases: `RuntimeError` + +Undefined error with the container + +#### _`exception{:python}`_`RegistryNotFoundError(error_log){:python}` + +Bases: `ConnectionRefusedError` + +The specified registry does not resolve to a valid IP address + +### _`exception{:python}`_`nf_core.download.DownloadError{:python}` + +Bases: `RuntimeError` + +A custom exception that is raised when nf-core download encounters a problem that we already took into consideration. +In this case, we do not want to print the traceback, but give the user some concise, helpful feedback instead. + +### _`class{:python}`_`nf_core.download.DownloadProgress(*columns: str | ProgressColumn, console: Console | None = None, auto_refresh: bool = True, refresh_per_second: float = 10, speed_estimate_period: float = 30.0, transient: bool = False, redirect_stdout: bool = True, redirect_stderr: bool = True, get_time: Callable[[], float] | None = None, disable: bool = False, expand: bool = False){:python}` + +Bases: `Progress` + +Custom Progress bar class, allowing us to have two progress +bars with different columns / layouts. + +#### `get_renderables(){:python}` + +Get a number of renderables for the progress display. + +### _`class{:python}`_`nf_core.download.DownloadWorkflow(pipeline=None, revision=None, outdir=None, compress_type=None, force=False, tower=False, download_configuration=None, container_system=None, container_library=None, container_cache_utilisation=None, container_cache_index=None, parallel_downloads=4){:python}` + +Bases: `object` + +Downloads a nf-core workflow from GitHub to the local file system. + +Can also download its Singularity container image if required. + +- **Parameters:** + - **pipeline** (_str_) – A nf-core pipeline name. + - **revision** (_List_\*\[**str**]\*) – The workflow revision to download, like 1.0. Defaults to None. + - **container** (_bool_) – Flag, if the Singularity container should be downloaded as well. Defaults to False. + - **tower** (_bool_) – Flag, to customize the download for Nextflow Tower (convert to git bare repo). Defaults to False. + - **outdir** (_str_) – Path to the local download directory. Defaults to None. + +#### `compress_download(){:python}` + +Take the downloaded files and make a compressed .tar.gz archive. + +#### `download_configs(){:python}` + +Downloads the centralised config profiles from nf-core/configs to `self.outdir`. + +#### `download_wf_files(revision, wf_sha, download_url){:python}` + +Downloads workflow files from GitHub to the `self.outdir`. + +#### `download_workflow(){:python}` + +Starts a nf-core workflow download. + +#### `download_workflow_static(){:python}` + +Downloads a nf-core workflow from GitHub to the local file system in a self-contained manner. + +#### `download_workflow_tower(location=None){:python}` + +Create a bare-cloned git repository of the workflow, so it can be launched with tw launch as pipeline + +#### `find_container_images(workflow_directory){:python}` + +Find container image names for workflow. + +Starts by using nextflow config to pull out any process.container +declarations. This works for DSL1. It should return a simple string with resolved logic, +but not always, e.g. not for differentialabundance 1.2.0 + +Second, we look for DSL2 containers. These can’t be found with +nextflow config at the time of writing, so we scrape the pipeline files. +This returns raw matches that will likely need to be cleaned. + +#### `get_revision_hash(){:python}` + +Find specified revision / branch hash + +#### `get_singularity_images(current_revision=''){:python}` + +Loop through container names and download Singularity images + +#### `prioritize_direct_download(container_list){:python}` + +Helper function that takes a list of container images (URLs and Docker URIs), +eliminates all Docker URIs for which also a URL is contained and returns the +cleaned and also deduplicated list. + +Conceptually, this works like so: + +Everything after the last Slash should be identical, e.g. “scanpy:1.7.2–pyhdfd78af_0” in +\[‘https://depot.galaxyproject.org/singularity/scanpy:1.7.2–pyhdfd78af\_0’, ‘biocontainers/scanpy:1.7.2–pyhdfd78af_0’] + +re.sub(‘.\*/(.\*)’,’1’,c) will drop everything up to the last slash from c (container_id) + +d.get(k:=re.sub(‘.\*/(.\*)’,’1’,c),’’) assigns the truncated string to k (key) and gets the +corresponding value from the dict if present or else defaults to “”. + +If the regex pattern matches, the original container_id will be assigned to the dict with the k key. +r”^$|(?!^http)” matches an empty string (we didn’t have it in the dict yet and want to keep it in either case) or +any string that does not start with http. Because if our current dict value already starts with http, +we want to keep it and not replace with with whatever we have now (which might be the Docker URI). + +A regex that matches http, r”^$|^http” could thus be used to prioritize the Docker URIs over http Downloads + +#### `prompt_compression_type(){:python}` + +Ask user if we should compress the downloaded files + +#### `prompt_config_inclusion(){:python}` + +Prompt for inclusion of institutional configurations + +#### `prompt_container_download(){:python}` + +Prompt whether to download container images or not + +#### `prompt_pipeline_name(){:python}` + +Prompt for the pipeline name if not set with a flag + +#### `prompt_revision(){:python}` + +Prompt for pipeline revision / branch +Prompt user for revision tag if ‘–revision’ was not set +If –tower is specified, allow to select multiple revisions +Also the static download allows for multiple revisions, but +we do not prompt this option interactively. + +#### `prompt_singularity_cachedir_creation(){:python}` + +Prompt about using $NXF_SINGULARITY_CACHEDIR if not already set + +#### `prompt_singularity_cachedir_remote(){:python}` + +Prompt about the index of a remote $NXF_SINGULARITY_CACHEDIR + +#### `prompt_singularity_cachedir_utilization(){:python}` + +Ask if we should _only_ use $NXF_SINGULARITY_CACHEDIR without copying into target + +#### `read_remote_containers(){:python}` + +Reads the file specified as index for the remote Singularity cache dir + +#### `rectify_raw_container_matches(raw_findings){:python}` + +Helper function to rectify the raw extracted container matches into fully qualified container names. +If multiple containers are found, any prefixed with http for direct download is prioritized + +Example syntax: + +Early DSL2: + +```groovy +if (workflow.containerEngine == 'singularity' && !params.singularity_pull_docker_container) { + container "https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0" +} else { + container "quay.io/biocontainers/fastqc:0.11.9--0" +} +``` + +Later DSL2: + +```groovy +container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + 'https://depot.galaxyproject.org/singularity/fastqc:0.11.9--0' : + 'biocontainers/fastqc:0.11.9--0' }" +``` + +Later DSL2, variable is being used: + +```groovy +container "${ workflow.containerEngine == 'singularity' && !task.ext.singularity_pull_docker_container ? + "https://depot.galaxyproject.org/singularity/${container_id}" : + "quay.io/biocontainers/${container_id}" }" + +container_id = 'mulled-v2-1fa26d1ce03c295fe2fdcf85831a92fbcbd7e8c2:afaaa4c6f5b308b4b6aa2dd8e99e1466b2a6b0cd-0' +``` + +DSL1 / Special case DSL2: + +```groovy +container "nfcore/cellranger:6.0.2" +``` + +#### `singularity_copy_cache_image(container, out_path, cache_path){:python}` + +Copy Singularity image from NXF_SINGULARITY_CACHEDIR to target folder. + +#### `singularity_download_image(container, out_path, cache_path, progress){:python}` + +Download a singularity image from the web. + +Use native Python to download the file. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `https://depot.galaxyproject.org/singularity/name:version` + - **out_path** (_str_) – The final target output path + - **cache_path** (_str_\*,\* _None_) – The NXF_SINGULARITY_CACHEDIR path if set, None if not + - **progress** (_Progress_) – Rich progress bar instance to add tasks to. + +#### `singularity_image_filenames(container){:python}` + +Check Singularity cache for image, copy to destination folder if found. + +- **Parameters:** + **container** (_str_) – A pipeline’s container name. Can be direct download URL + or a Docker Hub repository ID. +- **Returns:** + Returns True if we have the image in the target location. + : Returns a download path if not. +- **Return type:** + results (bool, str) + +#### `singularity_pull_image(container, out_path, cache_path, library, progress){:python}` + +Pull a singularity image using `singularity pull` + +Attempt to use a local installation of singularity to pull the image. + +- **Parameters:** + - **container** (_str_) – A pipeline’s container name. Usually it is of similar format + to `nfcore/name:version`. + - **library** (_list_ _of_ _str_) – A list of libraries to try for pulling the image. +- **Raises:** + **Various exceptions possible from subprocess execution** **of** **Singularity.** – + +#### `wf_use_local_configs(revision_dirname){:python}` + +Edit the downloaded nextflow.config file to use the local config files + +### _`class{:python}`_`nf_core.download.WorkflowRepo(remote_url, revision, commit, location=None, hide_progress=False, in_cache=True){:python}` + +Bases: `SyncedRepo` + +An object to store details about a locally cached workflow repository. + +Important Attributes: +: fullname: The full name of the repository, `nf-core/{self.pipelinename}`. +local_repo_dir (str): The local directory, where the workflow is cloned into. Defaults to `$HOME/.cache/nf-core/nf-core/{self.pipeline}`. + +#### `access(){:python}` + +#### `bare_clone(destination){:python}` + +#### `checkout(commit){:python}` + +Checks out the repository at the requested commit + +- **Parameters:** + **commit** (_str_) – Git SHA of the commit + +#### `get_remote_branches(remote_url){:python}` + +Get all branches from a remote repository + +- **Parameters:** + **remote_url** (_str_) – The git url to the remote repository +- **Returns:** + All branches found in the remote +- **Return type:** + (set\[str]) + +#### `retry_setup_local_repo(skip_confirm=False){:python}` + +#### `setup_local_repo(remote, location=None, in_cache=True){:python}` + +Sets up the local git repository. If the repository has been cloned previously, it +returns a git.Repo object of that clone. Otherwise it tries to clone the repository from +the provided remote URL and returns a git.Repo of the new clone. + +- **Parameters:** + - **remote** (_str_) – git url of remote + - **location** (_Path_) – location where the clone should be created/cached. + - **in_cache** (_bool_\*,\* _optional_) – Whether to clone the repository from the cache. Defaults to False. + +Sets self.repo + +#### `tidy_tags_and_branches(){:python}` + +Function to delete all tags and branches that are not of interest to the downloader. +This allows a clutter-free experience in Tower. The untagged commits are evidently still available. + +However, due to local caching, the downloader might also want access to revisions that had been deleted before. +In that case, don’t bother with re-adding the tags and rather download anew from Github. diff --git a/src/content/tools/docs/dev/api/index.md b/src/content/tools/docs/dev/api/index.md new file mode 100644 index 0000000000..4b230797b7 --- /dev/null +++ b/src/content/tools/docs/dev/api/index.md @@ -0,0 +1,15 @@ +# API Reference + +- [nf_core.bump_version](bump_version) +- [nf_core.create](create) +- [nf_core.download](download) +- [nf_core.launch](launch) +- [nf_core.licences](licences) +- [nf_core.lint](lint) +- [nf_core.list](list) +- [nf_core.modules](modules) +- [nf_core.params_file](params-file) +- [nf_core.schema](schema) +- [nf_core.subworkflows](subworkflows) +- [nf_core.sync](sync) +- [nf_core.utils](utils) diff --git a/src/content/tools/docs/dev/api/launch.md b/src/content/tools/docs/dev/api/launch.md new file mode 100644 index 0000000000..8affe0fa4d --- /dev/null +++ b/src/content/tools/docs/dev/api/launch.md @@ -0,0 +1,87 @@ +# nf_core.launch + +Launch a pipeline, interactively collecting params + +### _`class{:python}`_`nf_core.launch.Launch(pipeline=None, revision=None, command_only=False, params_in=None, params_out=None, save_all=False, show_hidden=False, url=None, web_id=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline + +#### `build_command(){:python}` + +Build the nextflow run command based on what we know + +#### `get_pipeline_schema(){:python}` + +Load and validate the schema from the supplied pipeline + +#### `get_web_launch_response(){:python}` + +Given a URL for a web-gui launch response, recursively query it until results are ready. + +#### `launch_pipeline(){:python}` + +#### `launch_web_gui(){:python}` + +Send schema to nf-core website and launch input GUI + +#### `launch_workflow(){:python}` + +Launch nextflow if required + +#### `merge_nxf_flag_schema(){:python}` + +Take the Nextflow flag schema and merge it with the pipeline schema + +#### `print_param_header(param_id, param_obj, is_group=False){:python}` + +#### `prompt_group(group_id, group_obj){:python}` + +Prompt for edits to a group of parameters (subschema in ‘definitions’) + +- **Parameters:** + - **group_id** – Paramater ID (string) + - **group_obj** – JSON Schema keys (dict) +- **Returns:** + val answers +- **Return type:** + Dict of param_id + +#### `prompt_param(param_id, param_obj, is_required, answers){:python}` + +Prompt for a single parameter + +#### `prompt_schema(){:python}` + +Go through the pipeline schema and prompt user to change defaults + +#### `prompt_web_gui(){:python}` + +Ask whether to use the web-based or cli wizard to collect params + +#### `sanitise_web_response(){:python}` + +The web builder returns everything as strings. +Use the functions defined in the cli wizard to convert to the correct types. + +#### `set_schema_inputs(){:python}` + +Take the loaded schema and set the defaults as the input parameters +If a nf_params.json file is supplied, apply these over the top + +#### `single_param_to_questionary(param_id, param_obj, answers=None, print_help=True){:python}` + +Convert a JSONSchema param to a Questionary question + +- **Parameters:** + - **param_id** – Parameter ID (string) + - **param_obj** – JSON Schema keys (dict) + - **answers** – Optional preexisting answers (dict) + - **print_help** – If description and help_text should be printed (bool) +- **Returns:** + Single Questionary dict, to be appended to questions list + +#### `strip_default_params(){:python}` + +Strip parameters if they have not changed from the default diff --git a/src/content/tools/docs/dev/api/licences.md b/src/content/tools/docs/dev/api/licences.md new file mode 100644 index 0000000000..578ddaae1d --- /dev/null +++ b/src/content/tools/docs/dev/api/licences.md @@ -0,0 +1,37 @@ +# nf_core.licences + +Lists software licences for a given workflow. + +### _`class{:python}`_`nf_core.licences.WorkflowLicences(pipeline){:python}` + +Bases: `object` + +A nf-core workflow licenses collection. + +Tries to retrieve the license information from all dependencies +of a given nf-core pipeline. + +A condensed overview with license per dependency can be printed out. + +- **Parameters:** + **pipeline** (_str_) – An existing nf-core pipeline name, like nf-core/hlatyping + or short hlatyping. + +#### `fetch_conda_licences(){:python}` + +Fetch package licences from Anaconda and PyPi. + +#### `get_environment_file(){:python}` + +Get the conda environment file for the pipeline + +#### `print_licences(){:python}` + +Prints the fetched license information. + +- **Parameters:** + **as_json** (_boolean_) – Prints the information in JSON. Defaults to False. + +#### `run_licences(){:python}` + +Run the nf-core licences action diff --git a/src/content/tools/docs/dev/api/lint.md b/src/content/tools/docs/dev/api/lint.md new file mode 100644 index 0000000000..abc826b708 --- /dev/null +++ b/src/content/tools/docs/dev/api/lint.md @@ -0,0 +1,134 @@ +# nf_core.lint + +:::tip +See the [Lint Tests](../pipeline_lint_tests/index) docs for information about specific linting functions. +::: + + + +Linting policy for nf-core pipeline projects. + +Tests Nextflow-based pipelines to check that they adhere to +the nf-core community guidelines. + +### `nf_core.lint.run_linting(pipeline_dir, release_mode=False, fix=(), key=(), show_passed=False, fail_ignored=False, fail_warned=False, sort_by='test', md_fn=None, json_fn=None, hide_progress=False){:python}` + +Runs all nf-core linting checks on a given Nextflow pipeline project +in either release mode or normal mode (default). Returns an object +of type [`PipelineLint`](#nf_core.lint.PipelineLint) after finished. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **release_mode** (_bool_) – Set this to True, if the linting should be run in the release mode. + See [`PipelineLint`](#nf_core.lint.PipelineLint) for more information. +- **Returns:** + An object of type [`PipelineLint`](#nf_core.lint.PipelineLint) that contains all the linting results. + An object of type `ComponentLint` that contains all the linting results for the modules. + An object of type `ComponentLint` that contains all the linting results for the subworkflows. + +### _`class{:python}`_`nf_core.lint.PipelineLint(wf_path, release_mode=False, fix=(), key=None, fail_ignored=False, fail_warned=False, hide_progress=False){:python}` + +Bases: [`Pipeline`](utils#nf_core.utils.Pipeline) + +Object to hold linting information and results. + +Inherits [`nf_core.utils.Pipeline`](utils#nf_core.utils.Pipeline) class. + +Use the [`PipelineLint._lint_pipeline()`](#nf_core.lint.PipelineLint._lint_pipeline) function to run lint tests. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `failed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `ignored{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `lint_config{:python}` + +The parsed nf-core linting config for this pipeline + +- **Type:** + dict + +#### `passed{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `release_mode{:python}` + +True, if you the to linting was run in release mode, False else. + +- **Type:** + bool + +#### `warned{:python}` + +A list of tuples of the form: `(, )` + +- **Type:** + list + +#### `_get_results_md(){:python}` + +Create a markdown file suitable for posting in a GitHub comment. + +- **Returns:** + Formatting markdown content +- **Return type:** + markdown (str) + +#### `_lint_pipeline(){:python}` + +Main linting function. + +Takes the pipeline directory as the primary input and iterates through +the different linting checks in order. Collects any warnings or errors +into object attributes: `passed`, `ignored`, `warned` and `failed`. + +#### `_print_results(show_passed){:python}` + +Print linting results to the command line. + +Uses the `rich` library to print a set of formatted tables to the command line +summarising the linting results. + +#### `_save_json_results(json_fn){:python}` + +Function to dump lint results to a JSON file for downstream use + +- **Parameters:** + **json_fn** (_str_) – File path to write JSON to. + +#### `_wrap_quotes(files: List[str] | List[Path] | Path){:python}` + +Helper function to take a list of filenames and format with markdown. + +- **Parameters:** + **files** (_list_) – + + List of filenames, eg: + + ```default + ['foo', 'bar', 'baz'] + ``` + +- **Returns:** + Formatted string of paths separated by word `or`, eg: + ```default + `foo` or bar` or `baz` + ``` +- **Return type:** + markdown (str) diff --git a/src/content/tools/docs/dev/api/list.md b/src/content/tools/docs/dev/api/list.md new file mode 100644 index 0000000000..8e58cd9854 --- /dev/null +++ b/src/content/tools/docs/dev/api/list.md @@ -0,0 +1,99 @@ +# nf_core.list + +Lists available nf-core pipelines and versions. + +### _`class{:python}`_`nf_core.list.LocalWorkflow(name){:python}` + +Bases: `object` + +Class to handle local workflows pulled by nextflow + +#### `get_local_nf_workflow_details(){:python}` + +Get full details about a local cached workflow + +### _`class{:python}`_`nf_core.list.RemoteWorkflow(data){:python}` + +Bases: `object` + +A information container for a remote workflow. + +- **Parameters:** + **data** (_dict_) – workflow information as they are retrieved from the GitHub repository REST API request + (). + +### _`class{:python}`_`nf_core.list.Workflows(filter_by=None, sort_by='release', show_archived=False){:python}` + +Bases: `object` + +Workflow container class. + +Is used to collect local and remote nf-core pipelines. Pipelines +can be sorted, filtered and compared. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + +#### `compare_remote_local(){:python}` + +Matches local to remote workflows. + +If a matching remote workflow is found, the local workflow’s Git commit hash is compared +with the latest one from remote. + +A boolean flag in `RemoteWorkflow.local_is_latest` is set to True, if the local workflow +is the latest. + +#### `filtered_workflows(){:python}` + +Filters remote workflows for keywords. + +- **Returns:** + Filtered remote workflows. +- **Return type:** + list + +#### `get_local_nf_workflows(){:python}` + +Retrieves local Nextflow workflows. + +Local workflows are stored in `self.local_workflows` list. + +#### `get_remote_workflows(){:python}` + +Retrieves remote workflows from [nf-co.re](https://nf-co.re). + +Remote workflows are stored in `self.remote_workflows` list. + +#### `print_json(){:python}` + +Dump JSON of all parsed information + +#### `print_summary(){:python}` + +Prints a summary of all pipelines. + +### `nf_core.list.get_local_wf(workflow: str | Path, revision=None){:python}` + +Check if this workflow has a local copy and use nextflow to pull it if not + +### `nf_core.list.list_workflows(filter_by=None, sort_by='release', as_json=False, show_archived=False){:python}` + +Prints out a list of all nf-core workflows. + +- **Parameters:** + - **filter_by** (_list_) – A list of strings that can be used for filtering. + - **sort_by** (_str_) – workflows can be sorted by keywords. Keyword must be one of + release (default), name, stars. + - **as_json** (_boolean_) – Set to true, if the lists should be printed in JSON. + +### `nf_core.list.pretty_date(time){:python}` + +Transforms a datetime object or a int() Epoch timestamp into a +pretty string like ‘an hour ago’, ‘Yesterday’, ‘3 months ago’, +‘just now’, etc + +Based on +Adapted by sven1103 diff --git a/src/content/tools/docs/dev/api/modules.md b/src/content/tools/docs/dev/api/modules.md new file mode 100644 index 0000000000..2dd47b6359 --- /dev/null +++ b/src/content/tools/docs/dev/api/modules.md @@ -0,0 +1 @@ +# nf_core.modules diff --git a/src/content/tools/docs/dev/api/params-file.md b/src/content/tools/docs/dev/api/params-file.md new file mode 100644 index 0000000000..cda56eb12c --- /dev/null +++ b/src/content/tools/docs/dev/api/params-file.md @@ -0,0 +1,82 @@ +# nf_core.params_file + +Create a YAML parameter file + +### _`class{:python}`_`nf_core.params_file.ParamsFileBuilder(pipeline=None, revision=None){:python}` + +Bases: `object` + +Class to hold config option to launch a pipeline. + +- **Parameters:** + - **pipeline** (_str_\*,\* _optional_) – Path to a local pipeline path or a remote pipeline. + - **revision** (_str_\*,\* _optional_) – Revision of the pipeline to use. + +#### `format_group(definition, show_hidden=False){:python}` + +Format a group of parameters of the schema as commented YAML. + +- **Parameters:** + - **definition** (_dict_) – Definition of the group from the schema + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for a group +- **Return type:** + str + +#### `format_param(name, properties, required_properties=(), show_hidden=False){:python}` + +Format a single parameter of the schema as commented YAML + +- **Parameters:** + - **name** (_str_) – Name of the parameter + - **properties** (_dict_) – Properties of the parameter + - **required_properties** (_list_) – List of required properties + - **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Section of a params-file.yml for given parameter + None: If the parameter is skipped because it is hidden and show_hidden is not set +- **Return type:** + str + +#### `generate_params_file(show_hidden=False){:python}` + +Generate the contents of a parameter template file. + +Assumes the pipeline has been fetched (if remote) and the schema loaded. + +- **Parameters:** + **show_hidden** (_bool_) – Whether to include hidden parameters +- **Returns:** + Formatted output for the pipeline schema +- **Return type:** + str + +#### `get_pipeline(){:python}` + +Prompt the user for a pipeline name and get the schema + +#### `write_params_file(output_fn='nf-params.yaml', show_hidden=False, force=False){:python}` + +Build a template file for the pipeline schema. + +- **Parameters:** + - **output_fn** (_str_\*,\* _optional_) – Filename to write the template to. + - **show_hidden** (_bool_\*,\* _optional_) – Include parameters marked as hidden in the output + - **force** (_bool_\*,\* _optional_) – Whether to overwrite existing output file. +- **Returns:** + True if the template was written successfully, False otherwise +- **Return type:** + bool + +### `nf_core.params_file._print_wrapped(text, fill_char='-', mode='both', width=80, indent=0, drop_whitespace=True){:python}` + +Helper function to format text for the params-file template. + +- **Parameters:** + - **text** (_str_) – Text to print + - **fill_char** (_str_\*,\* _optional_) – Character to use for creating dividers. Defaults to ‘-‘. + - **mode** (_str_\*,\* _optional_) – Where to place dividers. Defaults to “both”. + - **width** (_int_\*,\* _optional_) – Maximum line-width of the output text. Defaults to 80. + - **indent** (_int_\*,\* _optional_) – Number of spaces to indent the text. Defaults to 0. + - **drop_whitespace** (_bool_\*,\* _optional_) – Whether to drop whitespace from the start and end of lines. diff --git a/src/content/tools/docs/dev/api/schema.md b/src/content/tools/docs/dev/api/schema.md new file mode 100644 index 0000000000..d6cced3692 --- /dev/null +++ b/src/content/tools/docs/dev/api/schema.md @@ -0,0 +1,170 @@ +# nf_core.schema + +Code to deal with pipeline JSON Schema + +### _`class{:python}`_`nf_core.schema.PipelineSchema{:python}` + +Bases: `object` + +Class to generate a schema object with +functions to handle pipeline JSON Schema + +#### `add_schema_found_configs(){:python}` + +Add anything that’s found in the Nextflow params that’s missing in the pipeline schema +Update defaults if they have changed + +#### `build_schema(pipeline_dir, no_prompts, web_only, url){:python}` + +Interactively build a new pipeline schema for a pipeline + +#### `build_schema_param(p_val){:python}` + +Build a pipeline schema dictionary for an param interactively + +#### `check_for_input_mimetype(){:python}` + +Check that the input parameter has a mimetype + +Common mime types: + +- **Returns:** + The mimetype of the input parameter +- **Return type:** + mimetype (str) +- **Raises:** + **LookupError** – If the input parameter is not found or defined in the correct place + +#### `get_schema_defaults(){:python}` + +Generate set of default input parameters from schema. + +Saves defaults to self.schema_defaults +Returns count of how many parameters were found (with or without a default value) + +#### `get_schema_path(path, local_only=False, revision=None){:python}` + +Given a pipeline name, directory, or path, set self.schema_filename + +#### `get_schema_types(){:python}` + +Get a list of all parameter types in the schema + +#### `get_web_builder_response(){:python}` + +Given a URL for a Schema build response, recursively query it until results are ready. +Once ready, validate Schema and write to disk. + +#### `get_wf_params(){:python}` + +Load the pipeline parameter defaults using nextflow config +Strip out only the params. values and ignore anything that is not a flat variable + +#### `launch_web_builder(){:python}` + +Send pipeline schema to web builder and wait for response + +#### `load_input_params(params_path){:python}` + +Load a given a path to a parameters file (JSON/YAML) + +These should be input parameters used to run a pipeline with +the Nextflow -params-file option. + +#### `load_lint_schema(){:python}` + +Load and lint a given schema to see if it looks valid + +#### `load_schema(){:python}` + +Load a pipeline schema from a file + +#### `make_skeleton_schema(){:python}` + +Make a new pipeline schema from the template + +#### `markdown_param_table(properties, required, columns){:python}` + +Creates a markdown table for params from jsonschema properties section + +- **Parameters:** + - **properties** (_dict_) – A jsonschema properties dictionary + - **required** (_list_) – A list of the required fields. + Should come from the same level of the jsonschema as properties + - **columns** (_list_) – A list of columns to write +- **Returns:** + A string with the markdown table +- **Return type:** + str + +#### `markdown_to_html(markdown_str){:python}` + +Convert markdown to html + +#### `print_documentation(output_fn=None, format='markdown', force=False, columns=None){:python}` + +Prints documentation for the schema. + +#### `prompt_remove_schema_notfound_config(p_key){:python}` + +Check if a given key is found in the nextflow config params and prompt to remove it if note + +Returns True if it should be removed, False if not. + +#### `remove_schema_empty_definitions(){:python}` + +Go through top-level schema remove definitions that don’t have +any property attributes + +#### `remove_schema_notfound_configs(){:python}` + +Go through top-level schema and all definitions sub-schemas to remove +anything that’s not in the nextflow config. + +#### `remove_schema_notfound_configs_single_schema(schema){:python}` + +Go through a single schema / set of properties and strip out +anything that’s not in the nextflow config. + +Takes: Schema or sub-schema with properties key +Returns: Cleaned schema / sub-schema + +#### `sanitise_param_default(param){:python}` + +Given a param, ensure that the default value is the correct variable type + +#### `save_schema(suppress_logging=False){:python}` + +Save a pipeline schema to a file + +#### `schema_to_markdown(columns){:python}` + +Creates documentation for the schema in Markdown format. + +#### `validate_config_default_parameter(param, schema_param, config_default){:python}` + +Assure that default parameters in the nextflow.config are correctly set +by comparing them to their type in the schema + +#### `validate_default_params(){:python}` + +Check that all default parameters in the schema are valid +Ignores ‘required’ flag, as required parameters might have no defaults + +Additional check that all parameters have defaults in nextflow.config and that +these are valid and adhere to guidelines + +#### `validate_params(){:python}` + +Check given parameters against a schema and validate + +#### `validate_schema(schema=None){:python}` + +Check that the Schema is valid + +Returns: Number of parameters found + +#### `validate_schema_title_description(schema=None){:python}` + +Extra validation command for linting. +Checks that the schema “$id”, “title” and “description” attributes match the piipeline config. diff --git a/src/content/tools/docs/dev/api/subworkflows.md b/src/content/tools/docs/dev/api/subworkflows.md new file mode 100644 index 0000000000..19660c69cf --- /dev/null +++ b/src/content/tools/docs/dev/api/subworkflows.md @@ -0,0 +1 @@ +# nf_core.subworkflows diff --git a/src/content/tools/docs/dev/api/sync.md b/src/content/tools/docs/dev/api/sync.md new file mode 100644 index 0000000000..bc7034cf1c --- /dev/null +++ b/src/content/tools/docs/dev/api/sync.md @@ -0,0 +1,152 @@ +# nf_core.sync + +Synchronise a pipeline TEMPLATE branch with the template. + +### _`class{:python}`_`nf_core.sync.PipelineSync(pipeline_dir, from_branch=None, make_pr=False, gh_repo=None, gh_username=None, template_yaml_path=None){:python}` + +Bases: `object` + +Object to hold syncing information and results. + +- **Parameters:** + - **pipeline_dir** (_str_) – The path to the Nextflow pipeline root directory + - **from_branch** (_str_) – The branch to use to fetch config vars. If not set, will use current active branch + - **make_pr** (_bool_) – Set this to True to create a GitHub pull-request with the changes + - **gh_username** (_str_) – GitHub username + - **gh_repo** (_str_) – GitHub repository name + - **template_yaml_path** (_str_) – Path to template.yml file for pipeline creation settings. DEPRECATED + +#### `pipeline_dir{:python}` + +Path to target pipeline directory + +- **Type:** + str + +#### `from_branch{:python}` + +Repo branch to use when collecting workflow variables. Default: active branch. + +- **Type:** + str + +#### `original_branch{:python}` + +Repo branch that was checked out before we started. + +- **Type:** + str + +#### `made_changes{:python}` + +Whether making the new template pipeline introduced any changes + +- **Type:** + bool + +#### `make_pr{:python}` + +Whether to try to automatically make a PR on GitHub.com + +- **Type:** + bool + +#### `required_config_vars{:python}` + +List of nextflow variables required to make template pipeline + +- **Type:** + list + +#### `gh_username{:python}` + +GitHub username + +- **Type:** + str + +#### `gh_repo{:python}` + +GitHub repository name + +- **Type:** + str + +#### `checkout_template_branch(){:python}` + +Try to check out the origin/TEMPLATE in a new TEMPLATE branch. +If this fails, try to check out an existing local TEMPLATE branch. + +#### `close_open_pr(pr){:python}` + +Given a PR API response, add a comment and close. + +#### `close_open_template_merge_prs(){:python}` + +Get all template merging branches (starting with ‘nf-core-template-merge-‘) +and check for any open PRs from these branches to the self.from_branch +If open PRs are found, add a comment and close them + +#### `commit_template_changes(){:python}` + +If we have any changes with the new template files, make a git commit + +#### `create_merge_base_branch(){:python}` + +Create a new branch from the updated TEMPLATE branch +This branch will then be used to create the PR + +#### `delete_template_branch_files(){:python}` + +Delete all files in the TEMPLATE branch + +#### `get_wf_config(){:python}` + +Check out the target branch if requested and fetch the nextflow config. +Check that we have the required config variables. + +#### `inspect_sync_dir(){:python}` + +Takes a look at the target directory for syncing. Checks that it’s a git repo +and makes sure that there are no uncommitted changes. + +#### `make_pull_request(){:python}` + +Create a pull request to a base branch (default: dev), +from a head branch (default: TEMPLATE) + +Returns: An instance of class requests.Response + +#### `make_template_pipeline(){:python}` + +Delete all files and make a fresh template using the workflow variables + +#### `push_merge_branch(){:python}` + +Push the newly created merge branch to the remote repository + +#### `push_template_branch(){:python}` + +If we made any changes, push the TEMPLATE branch to the default remote +and try to make a PR. If we don’t have the auth token, try to figure out a URL +for the PR and print this to the console. + +#### `reset_target_dir(){:python}` + +Reset the target pipeline directory. Check out the original branch. + +#### `sync(){:python}` + +Find workflow attributes, create a new template pipeline on TEMPLATE + +### _`exception{:python}`_`nf_core.sync.PullRequestExceptionError{:python}` + +Bases: `Exception` + +Exception raised when there was an error creating a Pull-Request on GitHub.com + +### _`exception{:python}`_`nf_core.sync.SyncExceptionError{:python}` + +Bases: `Exception` + +Exception raised when there was an error with TEMPLATE branch synchronisation diff --git a/src/content/tools/docs/dev/api/utils.md b/src/content/tools/docs/dev/api/utils.md new file mode 100644 index 0000000000..4b4cae4ea6 --- /dev/null +++ b/src/content/tools/docs/dev/api/utils.md @@ -0,0 +1,408 @@ +# nf_core.utils + +Common utility functions for the nf-core python package. + +### _`class{:python}`_`nf_core.utils.GitHubAPISession{:python}` + +Bases: `CachedSession` + +Class to provide a single session for interacting with the GitHub API for a run. +Inherits the requests_cache.CachedSession and adds additional functionality, +such as automatically setting up GitHub authentication if we can. + +#### `get(url, **kwargs){:python}` + +Initialise the session if we haven’t already, then call the superclass get method. + +#### `lazy_init(){:python}` + +Initialise the object. + +Only do this when it’s actually being used (due to global import) + +#### `log_content_headers(request, post_data=None){:python}` + +Try to dump everything to the console, useful when things go wrong. + +#### `request_retry(url, post_data=None){:python}` + +Try to fetch a URL, keep retrying if we get a certain return code. + +Used in nf-core sync code because we get 403 errors: too many simultaneous requests +See + +#### `safe_get(url){:python}` + +Run a GET request, raise a nice exception with lots of logging if it fails. + +#### `setup_github_auth(auth=None){:python}` + +Try to automatically set up GitHub authentication + +### _`class{:python}`_`nf_core.utils.Pipeline(wf_path){:python}` + +Bases: `object` + +Object to hold information about a local pipeline. + +- **Parameters:** + **path** (_str_) – The path to the nf-core pipeline directory. + +#### `conda_config{:python}` + +The parsed conda configuration file content (`environment.yml`). + +- **Type:** + dict + +#### `conda_package_info{:python}` + +The conda package(s) information, based on the API requests to Anaconda cloud. + +- **Type:** + dict + +#### `nf_config{:python}` + +The Nextflow pipeline configuration file content. + +- **Type:** + dict + +#### `files{:python}` + +A list of files found during the linting process. + +- **Type:** + list + +#### `git_sha{:python}` + +The git sha for the repo commit / current GitHub pull-request ($GITHUB_PR_COMMIT) + +- **Type:** + str + +#### `minNextflowVersion{:python}` + +The minimum required Nextflow version to run the pipeline. + +- **Type:** + str + +#### `wf_path{:python}` + +Path to the pipeline directory. + +- **Type:** + str + +#### `pipeline_name{:python}` + +The pipeline name, without the nf-core tag, for example hlatyping. + +- **Type:** + str + +#### `schema_obj{:python}` + +A `PipelineSchema` object + +- **Type:** + obj + +#### `_fp(fn){:python}` + +Convenience function to get full path to a file in the pipeline + +#### `_list_files(){:python}` + +Get a list of all files in the pipeline + +#### `_load(){:python}` + +Run core load functions + +#### `_load_conda_environment(){:python}` + +Try to load the pipeline environment.yml file, if it exists + +#### `_load_pipeline_config(){:python}` + +Get the nextflow config for this pipeline + +Once loaded, set a few convienence reference class attributes + +### _`class{:python}`_`nf_core.utils.SingularityCacheFilePathValidator{:python}` + +Bases: `Validator` + +Validator for file path specified as –singularity-cache-index argument in nf-core download + +#### `_abc_impl{:python}`_= <\_abc.\_abc_data object>_ + +#### `validate(value){:python}` + +Validate the input. +If invalid, this should raise a `ValidationError`. + +- **Parameters:** + **document** – `Document` instance. + +### `nf_core.utils.anaconda_package(dep, dep_channels=None){:python}` + +Query conda package information. + +Sends a HTTP GET request to the Anaconda remote API. + +- **Parameters:** + - **dep** (_str_) – A conda package name. + - **dep_channels** (_list_) – list of conda channels to use +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.check_if_outdated(current_version=None, remote_version=None, source_url='https://nf-co.re/tools_version'){:python}` + +Check if the current version of nf-core is outdated + +### `nf_core.utils.custom_yaml_dumper(){:python}` + +Overwrite default PyYAML output to make Prettier YAML linting happy + +### `nf_core.utils.determine_base_dir(directory='.'){:python}` + +### `nf_core.utils.fetch_remote_version(source_url){:python}` + +### `nf_core.utils.fetch_wf_config(wf_path, cache_config=True){:python}` + +Uses Nextflow to retrieve the the configuration variables +from a Nextflow workflow. + +- **Parameters:** + - **wf_path** (_str_) – Nextflow workflow file system path. + - **cache_config** (_bool_) – cache configuration or not (def. True) +- **Returns:** + Workflow configuration settings. +- **Return type:** + dict + +### `nf_core.utils.file_md5(fname){:python}` + +Calculates the md5sum for a file on the disk. + +- **Parameters:** + **fname** (_str_) – Path to a local file. + +### `nf_core.utils.get_biocontainer_tag(package, version){:python}` + +Given a bioconda package and version, looks for Docker and Singularity containers +using the biocontaineres API, e.g.: +/{tool}/versions/{tool}-{version} +Returns the most recent container versions by default. +:param package: A bioconda package name. +:type package: str +:param version: Version of the bioconda package +:type version: str + +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** **or** **gives an unexpected status code** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** **(\*\***404\***\*)** – + +### `nf_core.utils.get_first_available_path(directory, paths){:python}` + +### `nf_core.utils.get_repo_releases_branches(pipeline, wfs){:python}` + +Fetches details of a nf-core workflow to download. + +- **Parameters:** + - **pipeline** (_str_) – GitHub repo username/repo + - **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + Array of releases, Array of branches +- **Return type:** + wf_releases, wf_branches (tuple) +- **Raises:** + **LockupError**\*\*,\*\* **if the pipeline can not be found.** – + +### `nf_core.utils.is_file_binary(path){:python}` + +Check file path to see if it is a binary file + +### `nf_core.utils.is_pipeline_directory(wf_path){:python}` + +Checks if the specified directory have the minimum required files +(‘main.nf’, ‘nextflow.config’) for a pipeline directory + +- **Parameters:** + **wf_path** (_str_) – The directory to be inspected +- **Raises:** + **UserWarning** – If one of the files are missing + +### `nf_core.utils.is_relative_to(path1, path2){:python}` + +Checks if a path is relative to another. + +Should mimic Path.is_relative_to which not available in Python < 3.9 + +path1 (Path | str): The path that could be a subpath +path2 (Path | str): The path the could be the superpath + +### `nf_core.utils.load_tools_config(directory: str | Path = '.'){:python}` + +Parse the nf-core.yml configuration file + +Look for a file called either .nf-core.yml or .nf-core.yaml + +Also looks for the deprecated file .nf-core-lint.yml/yaml and issues +a warning that this file will be deprecated in the future + +Returns the loaded config dict or False, if the file couldn’t be loaded + +### `nf_core.utils.nested_delitem(d, keys){:python}` + +Deletes a key from a nested dictionary + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse, deleting the final one + +### `nf_core.utils.nested_setitem(d, keys, value){:python}` + +Sets the value in a nested dict using a list of keys to traverse + +- **Parameters:** + - **d** (_dict_) – the nested dictionary to traverse + - **keys** (_list_\*\[**Any**]\*) – A list of keys to iteratively traverse + - **value** (_Any_) – The value to be set for the last key in the chain + +### `nf_core.utils.parse_anaconda_licence(anaconda_response, version=None){:python}` + +Given a response from the anaconda API using anaconda_package, parse the software licences. + +Returns: Set of licence types + +### `nf_core.utils.pip_package(dep){:python}` + +Query PyPI package information. + +Sends a HTTP GET request to the PyPI remote API. + +- **Parameters:** + **dep** (_str_) – A PyPI package name. +- **Raises:** + - **A LookupError**\*\*,\*\* **if the connection fails** **or** **times out** – + - **A ValueError**\*\*,\*\* **if the package name can not be found** – + +### `nf_core.utils.plural_es(list_or_int){:python}` + +Return a ‘es’ if the input is not one or has not the length of one. + +### `nf_core.utils.plural_s(list_or_int){:python}` + +Return an s if the input is not one or has not the length of one. + +### `nf_core.utils.plural_y(list_or_int){:python}` + +Return ‘ies’ if the input is not one or has not the length of one, else ‘y’. + +### `nf_core.utils.poll_nfcore_web_api(api_url, post_data=None){:python}` + +Poll the nf-core website API + +Takes argument api_url for URL + +Expects API reponse to be valid JSON and contain a top-level ‘status’ key. + +### `nf_core.utils.prompt_pipeline_release_branch(wf_releases, wf_branches, multiple=False){:python}` + +Prompt for pipeline release / branch + +- **Parameters:** + - **wf_releases** (_array_) – Array of repo releases as returned by the GitHub API + - **wf_branches** (_array_) – Array of repo branches, as returned by the GitHub API + - **multiple** (_bool_) – Allow selection of multiple releases & branches (for Tower) +- **Returns:** + Selected release / branch name +- **Return type:** + choice (str) + +### `nf_core.utils.prompt_remote_pipeline_name(wfs){:python}` + +Prompt for the pipeline name with questionary + +- **Parameters:** + **wfs** – A nf_core.list.Workflows() object, where get_remote_workflows() has been called. +- **Returns:** + GitHub repo - username/repo +- **Return type:** + pipeline (str) +- **Raises:** + **AssertionError**\*\*,\*\* **if pipeline cannot be found** – + +### `nf_core.utils.rich_force_colors(){:python}` + +Check if any environment variables are set to force Rich to use coloured output + +### `nf_core.utils.run_cmd(executable: str, cmd: str){:python}` + +Run a specified command and capture the output. Handle errors nicely. + +### `nf_core.utils.set_wd(path: Path){:python}` + +Sets the working directory for this context. + +- **Parameters:** + **path** (_Path_) – Path to the working directory to be used inside this context. + +### `nf_core.utils.setup_nfcore_cachedir(cache_fn: str | Path){:python}` + +Sets up local caching for caching files between sessions. + +### `nf_core.utils.setup_nfcore_dir(){:python}` + +Creates a directory for files that need to be kept between sessions + +Currently only used for keeping local copies of modules repos + +### `nf_core.utils.setup_requests_cachedir(){:python}` + +Sets up local caching for faster remote HTTP requests. + +Caching directory will be set up in the user’s home directory under +a .config/nf-core/cache\_\* subdir. + +Uses requests_cache monkey patching. +Also returns the config dict so that we can use the same setup with a Session. + +### `nf_core.utils.sort_dictionary(d){:python}` + +Sorts a nested dictionary recursively + +### `nf_core.utils.strip_ansi_codes(string, replace_with=''){:python}` + +Strip ANSI colouring codes from a string to return plain text. + +From Stack Overflow: + +### `nf_core.utils.validate_file_md5(file_name, expected_md5hex){:python}` + +Validates the md5 checksum of a file on disk. + +- **Parameters:** + - **file_name** (_str_) – Path to a local file. + - **expected** (_str_) – The expected md5sum. +- **Raises:** + **IOError**\*\*,\*\* **if the md5sum does not match the remote sum.** – + +### `nf_core.utils.wait_cli_function(poll_func, refresh_per_second=20){:python}` + +Display a command-line spinner while calling a function repeatedly. + +Keep waiting until that function returns True + +- **Parameters:** + - **poll_func** (_function_) – Function to call + - **refresh_per_second** (_int_) – Refresh this many times per second. Default: 20. +- **Returns:** + None. Just sits in an infite loop until the function returns True. diff --git a/src/content/tools/docs/dev/index.md b/src/content/tools/docs/dev/index.md new file mode 100644 index 0000000000..a483a5b647 --- /dev/null +++ b/src/content/tools/docs/dev/index.md @@ -0,0 +1,10 @@ +# nf-core/tools documentation + +This documentation is for the `nf-core/tools` package. + +## Contents + +- [Pipeline code lint tests](pipeline_lint_tests/index) (run by `nf-core lint`) +- [Module code lint tests](module_lint_tests/index) (run by `nf-core modules lint`) +- [Subworkflow code lint tests](subworkflow_lint_tests/index) (run by `nf-core subworkflows lint`) +- [nf-core/tools Python package API reference](api/index) diff --git a/src/content/tools/docs/dev/module_lint_tests/index.md b/src/content/tools/docs/dev/module_lint_tests/index.md new file mode 100644 index 0000000000..9d3660155b --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/index.md @@ -0,0 +1,10 @@ +# Module lint tests + +- [main_nf](main_nf) +- [meta_yml](meta_yml) +- [module_changes](module_changes) +- [module_deprecations](module_deprecations) +- [module_patch](module_patch) +- [module_tests](module_tests) +- [module_todos](module_todos) +- [module_version](module_version) diff --git a/src/content/tools/docs/dev/module_lint_tests/main_nf.md b/src/content/tools/docs/dev/module_lint_tests/main_nf.md new file mode 100644 index 0000000000..aa42cef6d7 --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/main_nf.md @@ -0,0 +1,20 @@ +# main_nf + +#### `ModuleLint.main_nf(module, fix_version, registry, progress_bar){:python}` + +Lint a `main.nf` module file + +Can also be used to lint local module files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- Software versions and containers are valid +- The module has a process label and it is among + the standard ones. +- If a `meta` map is defined as one of the modules + inputs it should be defined as one of the outputs, + and be correctly configured in the `saveAs` function. +- The module script section should contain definitions + of `software` and `prefix` diff --git a/src/content/tools/docs/dev/module_lint_tests/meta_yml.md b/src/content/tools/docs/dev/module_lint_tests/meta_yml.md new file mode 100644 index 0000000000..62ecd31ffa --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/meta_yml.md @@ -0,0 +1,29 @@ +# meta_yml + +#### `ModuleLint.meta_yml(module: NFCoreComponent){:python}` + +Lint a `meta.yml` file + +The lint test checks that the module has +a `meta.yml` file and that it follows the +JSON schema defined in the `modules/meta-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the module name +and module input is consistent between the +`meta.yml` and the `main.nf`. + +If the module has inputs or outputs, they are expected to be +formatted as: + +```groovy +tuple val(foo) path(bar) +val foo +path foo +``` + +or permutations of the above. + +- **Parameters:** + - **module_lint_object** (_ComponentLint_) – The lint object for the module + - **module** (_NFCoreComponent_) – The module to lint diff --git a/src/content/tools/docs/dev/module_lint_tests/module_changes.md b/src/content/tools/docs/dev/module_lint_tests/module_changes.md new file mode 100644 index 0000000000..3e8a7de3c4 --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/module_changes.md @@ -0,0 +1,14 @@ +# module_changes + +#### `ModuleLint.module_changes(module){:python}` + +Checks whether installed nf-core modules have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every module +and compares them to the local copies + +If the module has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/dev/module_lint_tests/module_deprecations.md b/src/content/tools/docs/dev/module_lint_tests/module_deprecations.md new file mode 100644 index 0000000000..763df0ab8a --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/module_deprecations.md @@ -0,0 +1,5 @@ +# module_deprecations + +#### `ModuleLint.module_deprecations(module){:python}` + +Check that the modules are up to the latest nf-core standard diff --git a/src/content/tools/docs/dev/module_lint_tests/module_patch.md b/src/content/tools/docs/dev/module_lint_tests/module_patch.md new file mode 100644 index 0000000000..5d38a40d31 --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/module_patch.md @@ -0,0 +1,8 @@ +# module_patch + +#### `ModuleLint.module_patch(module: NFCoreComponent){:python}` + +Lint a patch file found in a module + +Checks that the file name is well formed, and that +the patch can be applied in reverse with the correct result. diff --git a/src/content/tools/docs/dev/module_lint_tests/module_tests.md b/src/content/tools/docs/dev/module_lint_tests/module_tests.md new file mode 100644 index 0000000000..df2872f29e --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/module_tests.md @@ -0,0 +1,8 @@ +# module_tests + +#### `ModuleLint.module_tests(module: NFCoreComponent){:python}` + +Lint the tests of a module in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. diff --git a/src/content/tools/docs/dev/module_lint_tests/module_todos.md b/src/content/tools/docs/dev/module_lint_tests/module_todos.md new file mode 100644 index 0000000000..f4db6824f9 --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/module_todos.md @@ -0,0 +1,28 @@ +# module_todos + +#### `ModuleLint.module_todos(module){:python}` + +Look for TODO statements in the module files + +The nf-core module template contains a number of comment lines to help developers +of new modules know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the module and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/dev/module_lint_tests/module_version.md b/src/content/tools/docs/dev/module_lint_tests/module_version.md new file mode 100644 index 0000000000..62581d3d81 --- /dev/null +++ b/src/content/tools/docs/dev/module_lint_tests/module_version.md @@ -0,0 +1,9 @@ +# module_version + +#### `ModuleLint.module_version(module){:python}` + +Verifies that the module has a version specified in the `modules.json` file + +It checks whether the module has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the module available. diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/actions_awsfulltest.md b/src/content/tools/docs/dev/pipeline_lint_tests/actions_awsfulltest.md new file mode 100644 index 0000000000..9973427a79 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/actions_awsfulltest.md @@ -0,0 +1,30 @@ +# actions_awsfulltest + +#### `PipelineLint.actions_awsfulltest(){:python}` + +Checks the GitHub Actions awsfulltest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on full size datasets on AWS. +This should ensure that the pipeline runs as expected on AWS and provide a resource estimation. + +The GitHub Actions workflow is called `awsfulltest.yml`, and it can be found in the `.github/workflows/` directory. + +:::warning +This workflow incurs AWS costs, therefore it should only be triggered for pipeline releases: +`release` (after the pipeline release) and `workflow_dispatch`. +::: + +:::note +You can manually trigger the AWS tests by going to the Actions tab on the pipeline GitHub repository and selecting the +nf-core AWS full size tests workflow on the left. +::: + +:::note +For tests on full data prior to release, [Nextflow Tower](https://tower.nf) launch feature can be employed. +::: + +The `.github/workflows/awsfulltest.yml` file is tested for the following: + +- Must be turned on `workflow_dispatch`. +- Must be turned on for `release` with `types: [published]` +- Should run the profile `test_full` that should be edited to provide the links to full-size datasets. If it runs the profile `test`, a warning is given. diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/actions_awstest.md b/src/content/tools/docs/dev/pipeline_lint_tests/actions_awstest.md new file mode 100644 index 0000000000..4e4698c4e9 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/actions_awstest.md @@ -0,0 +1,24 @@ +# actions_awstest + +#### `PipelineLint.actions_awstest(){:python}` + +Checks the GitHub Actions awstest is valid. + +In addition to small test datasets run on GitHub Actions, we provide the possibility of testing the pipeline on AWS. +This should ensure that the pipeline runs as expected on AWS (which often has its own unique edge cases). + +:::warning +Running tests on AWS incurs costs, so these tests are not triggered automatically. +Instead, they use the `workflow_dispatch` trigger, which allows for manual triggering +of the workflow when testing on AWS is desired. +::: + +:::note +You can trigger the tests by going to the Actions tab on the pipeline GitHub repository +and selecting the nf-core AWS test workflow on the left. +::: + +The `.github/workflows/awstest.yml` file is tested for the following: + +- Must _not_ be turned on for `push` or `pull_request`. +- Must be turned on for `workflow_dispatch`. diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/actions_ci.md b/src/content/tools/docs/dev/pipeline_lint_tests/actions_ci.md new file mode 100644 index 0000000000..aec43b98fd --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/actions_ci.md @@ -0,0 +1,36 @@ +# actions_ci + +#### `PipelineLint.actions_ci(){:python}` + +Checks that the GitHub Actions pipeline CI (Continuous Integration) workflow is valid. + +The `.github/workflows/ci.yml` GitHub Actions workflow runs the pipeline on a minimal test +dataset using `-profile test` to check that no breaking changes have been introduced. +Final result files are not checked, just that the pipeline exists successfully. + +This lint test checks this GitHub Actions workflow file for the following: + +- Workflow must be triggered on the following events: + ```yaml + on: + push: + branches: + - dev + pull_request: + release: + types: [published] + ``` +- The minimum Nextflow version specified in the pipeline’s `nextflow.config` matches that defined by `NXF_VER` in the test matrix: + + ```yaml + strategy: + matrix: + # Nextflow versions: check pipeline minimum and current latest + NXF_VER: ['19.10.0', ''] + ``` + + :::note + These `matrix` variables run the test workflow twice, varying the `NXF_VER` variable each time. + This is used in the `nextflow run` commands to test the pipeline with both the latest available version + of the pipeline (`''`) and the stated minimum required version. + ::: diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/actions_schema_validation.md b/src/content/tools/docs/dev/pipeline_lint_tests/actions_schema_validation.md new file mode 100644 index 0000000000..518037bc74 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/actions_schema_validation.md @@ -0,0 +1,12 @@ +# actions_schema_validation + +#### `PipelineLint.actions_schema_validation(){:python}` + +Checks that the GitHub Action workflow yml/yaml files adhere to the correct schema + +nf-core pipelines use GitHub actions workflows to run CI tests, check formatting and also linting, among others. +These workflows are defined by `yml` scripts in `.github/workflows/`. This lint test verifies that these scripts are valid +by comparing them against the [JSON schema for GitHub workflows](https://json.schemastore.org/github-workflow). + +To pass this test, make sure that all your workflows contain the required properties `on` and `jobs` and that +all other properties are of the correct type, as specified in the schema (link above). diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/files_exist.md b/src/content/tools/docs/dev/pipeline_lint_tests/files_exist.md new file mode 100644 index 0000000000..b5b230ab71 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/files_exist.md @@ -0,0 +1,116 @@ +# files_exist + +#### `PipelineLint.files_exist(){:python}` + +Checks a given pipeline directory for required files. + +Iterates through the pipeline’s directory content and checks that specified +files are either present or absent, as required. + +:::note +This test raises an `AssertionError` if neither `nextflow.config` or `main.nf` are found. +If these files are not found then this cannot be a Nextflow pipeline and something has gone badly wrong. +All lint tests are stopped immediately with a critical error message. +::: + +Files that _must_ be present: + +```bash +.gitattributes +.gitignore +.nf-core.yml +.editorconfig +.prettierignore +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/ci.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +[LICENSE, LICENSE.md, LICENCE, LICENCE.md] # NB: British / American spelling +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +conf/modules.config +conf/test.config +conf/test_full.config +CHANGELOG.md +CITATIONS.md +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/output.md +docs/README.md +docs/usage.md +lib/NfcoreTemplate.groovy +lib/Utils.groovy +lib/WorkflowMain.groovy +nextflow_schema.json +nextflow.config +README.md +``` + +Files that _should_ be present: + +```bash +main.nf +assets/multiqc_config.yml +conf/base.config +conf/igenomes.config +.github/workflows/awstest.yml +.github/workflows/awsfulltest.yml +lib/WorkflowPIPELINE.groovy +pyproject.toml +``` + +Files that _must not_ be present, due to being renamed or removed in the template: + +```bash +Singularity +parameters.settings.json +pipeline_template.yml # saving information in .nf-core.yml +.nf-core.yaml # NB: Should be yml, not yaml +bin/markdown_to_html.r +conf/aws.config +.github/workflows/push_dockerhub.yml +.github/ISSUE_TEMPLATE/bug_report.md +.github/ISSUE_TEMPLATE/feature_request.md +docs/images/nf-core-PIPELINE_logo.png +.markdownlint.yml +.yamllint.yml +lib/Checks.groovy +lib/Completion.groovy +lib/Workflow.groovy +``` + +Files that _should not_ be present: + +```bash +.travis.yml +``` + +Files that _must not_ be present if a certain entry is present in `nextflow.config`: + +```bash +lib/nfcore_external_java_deps.jar # if "nf-validation" is in nextflow.config +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_exist` key as follows in your `.nf-core.yml` config file. For example: + +```yaml + +``` + +::: + +lint: +: files_exist: +: - assets/multiqc_config.yml diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/files_unchanged.md b/src/content/tools/docs/dev/pipeline_lint_tests/files_unchanged.md new file mode 100644 index 0000000000..ccb1541ae3 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/files_unchanged.md @@ -0,0 +1,55 @@ +# files_unchanged + +#### `PipelineLint.files_unchanged(){:python}` + +Checks that certain pipeline files are not modified from template output. + +Iterates through the pipeline’s directory content and compares specified files +against output from the template using the pipeline’s metadata. File content +should not be modified / missing. + +Files that must be unchanged: + +```default +.gitattributes +.prettierrc.yml +.github/.dockstore.yml +.github/CONTRIBUTING.md +.github/ISSUE_TEMPLATE/bug_report.yml +.github/ISSUE_TEMPLATE/config.yml +.github/ISSUE_TEMPLATE/feature_request.yml +.github/PULL_REQUEST_TEMPLATE.md +.github/workflows/branch.yml +.github/workflows/linting_comment.yml +.github/workflows/linting.yml +assets/email_template.html +assets/email_template.txt +assets/nf-core-PIPELINE_logo_light.png +assets/sendmail_template.txt +CODE_OF_CONDUCT.md +docs/images/nf-core-PIPELINE_logo_light.png +docs/images/nf-core-PIPELINE_logo_dark.png +docs/README.md' +lib/NfcoreTemplate.groovy +['LICENSE', 'LICENSE.md', 'LICENCE', 'LICENCE.md'], # NB: British / American spelling +``` + +Files that can have additional content but must include the template contents: + +```default +.gitignore +.prettierignore +pyproject.toml +``` + +:::note +You can configure the `nf-core lint` tests to ignore any of these checks by setting +the `files_unchanged` key as follows in your `.nf-core.yml` config file. For example: + +```yaml +lint: + files_unchanged: + - .github/workflows/branch.yml +``` + +::: diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/index.md b/src/content/tools/docs/dev/pipeline_lint_tests/index.md new file mode 100644 index 0000000000..35f92b4e9a --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/index.md @@ -0,0 +1,22 @@ +# Pipeline lint tests + +- [actions_awsfulltest](actions_awsfulltest) +- [actions_awstest](actions_awstest) +- [actions_ci](actions_ci) +- [actions_schema_validation](actions_schema_validation) +- [files_exist](files_exist) +- [files_unchanged](files_unchanged) +- [merge_markers](merge_markers) +- [nextflow_config](modules_json) +- [modules_structure](modules_structure) +- [multiqc_config](multiqc_config) +- [nextflow_config](nextflow_config) +- [pipeline_name_conventions](pipeline_name_conventions) +- [pipeline_todos](pipeline_todos) +- [readme](readme) +- [schema_description](schema_description) +- [schema_lint](schema_lint) +- [schema_params](schema_params) +- [system_exit](system_exit) +- [template_strings](template_strings) +- [version_consistency](version_consistency) diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/merge_markers.md b/src/content/tools/docs/dev/pipeline_lint_tests/merge_markers.md new file mode 100644 index 0000000000..2f6aef0aa4 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/merge_markers.md @@ -0,0 +1,28 @@ +# merge_markers + +#### `PipelineLint.merge_markers(){:python}` + +Check for remaining merge markers. + +This test looks for remaining merge markers in the code, e.g.: +`>>>>>>>` or `<<<<<<<` + +:::note +You can choose to ignore this lint tests by editing the file called +`.nf-core.yml` in the root of your pipeline and setting the test to false: + +```yaml +lint: + merge_markers: False +``` + +::: + +To disable this test only for specific files, you can specify a list of file paths to ignore. +For example, to ignore a pdf you added to the docs: + +```yaml +lint: + merge_markers: + - docs/my_pdf.pdf +``` diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/modules_json.md b/src/content/tools/docs/dev/pipeline_lint_tests/modules_json.md new file mode 100644 index 0000000000..0a66db5a23 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/modules_json.md @@ -0,0 +1,10 @@ +# nextflow_config + +#### `PipelineLint.modules_json(){:python}` + +Make sure all modules described in the `modules.json` file are actually installed + +Every module installed from `nf-core/modules` must have an entry in the `modules.json` file +with an associated version commit hash. + +- Failure: If module entries are found in `modules.json` for modules that are not installed diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/modules_structure.md b/src/content/tools/docs/dev/pipeline_lint_tests/modules_structure.md new file mode 100644 index 0000000000..4e08c24956 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/modules_structure.md @@ -0,0 +1,15 @@ +# modules_structure + +#### `PipelineLint.modules_structure(){:python}` + +Check that the structure of the modules directory in a pipeline is the correct one: + +```bash +modules/nf-core/TOOL/SUBTOOL +``` + +Prior to nf-core/tools release 2.6 the directory structure had an additional level of nesting: + +```bash +modules/nf-core/modules/TOOL/SUBTOOL +``` diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/multiqc_config.md b/src/content/tools/docs/dev/pipeline_lint_tests/multiqc_config.md new file mode 100644 index 0000000000..8cc900d36a --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/multiqc_config.md @@ -0,0 +1,19 @@ +# multiqc_config + +#### `PipelineLint.multiqc_config(){:python}` + +Make sure basic multiQC plugins are installed and plots are exported +Basic template: + +```yaml +report_comment: > + This report has been generated by the nf-core/quantms + analysis pipeline. For information about how to interpret these results, please see the + documentation. +report_section_order: + software_versions: + order: -1000 + nf-core-quantms-summary: + order: -1001 +export_plots: true +``` diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/nextflow_config.md b/src/content/tools/docs/dev/pipeline_lint_tests/nextflow_config.md new file mode 100644 index 0000000000..ba2ade27c8 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/nextflow_config.md @@ -0,0 +1,114 @@ +# nextflow_config + +#### `PipelineLint.nextflow_config(){:python}` + +Checks the pipeline configuration for required variables. + +All nf-core pipelines are required to be configured with a minimal set of variable +names. This test fails or throws warnings if required variables are not set. + +:::note +These config variables must be set in `nextflow.config` or another config +file imported from there. Any variables set in nextflow script files (eg. `main.nf`) +are not checked and will be assumed to be missing. +::: + +**The following variables fail the test if missing:** + +- `params.outdir`: A directory in which all pipeline results should be saved +- `manifest.name`: The pipeline name. Should begin with `nf-core/` +- `manifest.description`: A description of the pipeline +- `manifest.version` + - The version of this pipeline. This should correspond to a [GitHub release](https://help.github.com/articles/creating-releases/). + - If `--release` is set when running `nf-core lint`, the version number must not contain the string `dev` + - If `--release` is \_not\_ set, the version should end in `dev` (warning triggered if not) +- `manifest.nextflowVersion` + - The minimum version of Nextflow required to run the pipeline. + - Should be `>=` or `!>=` and a version number, eg. `manifest.nextflowVersion = '>=0.31.0'` (see [Nextflow documentation](https://www.nextflow.io/docs/latest/config.html#scope-manifest)) + - `>=` warns about old versions but tries to run anyway, `!>=` fails for old versions. Only use the latter if you _know_ that the pipeline will certainly fail before this version. + - This should correspond to the `NXF_VER` version tested by GitHub Actions. +- `manifest.homePage` + - The homepage for the pipeline. Should be the nf-core GitHub repository URL, + so beginning with `https://github.com/nf-core/` +- `timeline.enabled`, `trace.enabled`, `report.enabled`, `dag.enabled` + - The nextflow timeline, trace, report and DAG should be enabled by default (set to `true`) +- `process.cpus`, `process.memory`, `process.time` + - Default CPUs, memory and time limits for tasks +- `params.input` + - Input parameter to specify input data, specify this to avoid a warning + - Typical usage: + - `params.input`: Input data that is not NGS sequencing data +- `params.custom_config_version` + > - Should always be set to default value `master` +- `params.custom_config_base` + + > - Should always be set to default value: + + > `https://raw.githubusercontent.com/nf-core/configs/${params.custom_config_version}` + +- `params.validationShowHiddenParams` + > - Determines whether boilerplate params are showed by schema. Set to `false` by default +- `params.validationSchemaIgnoreParams` + > - A comma separated string of inputs the schema validation should ignore. + +**The following variables throw warnings if missing:** + +- `manifest.mainScript`: The filename of the main pipeline script (should be `main.nf`) +- `timeline.file`, `trace.file`, `report.file`, `dag.file` + - Default filenames for the timeline, trace and report + - The DAG file path should end with `.svg` (If Graphviz is not installed, Nextflow will generate a `.dot` file instead) + +**The following variables are depreciated and fail the test if they are still present:** + +- `params.version`: The old method for specifying the pipeline version. Replaced by `manifest.version` +- `params.nf_required_version`: The old method for specifying the minimum Nextflow version. Replaced by `manifest.nextflowVersion` +- `params.container`: The old method for specifying the dockerhub container address. Replaced by `process.container` +- `igenomesIgnore`: Changed to `igenomes_ignore` + > :::note + > The `snake_case` convention should now be used when defining pipeline parameters + > ::: + +**The following Nextflow syntax is depreciated and fails the test if present:** + +- Process-level configuration syntax still using the old Nextflow syntax, for example: `process.$fastqc` instead of `process withName:'fastqc'`. + +:::note +You can choose to ignore tests for the presence or absence of specific config variables +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config variables that should be ignored. For example: + +```yaml +lint: + nextflow_config: + - params.input +``` + +::: + +The other checks in this test (depreciated syntax etc) can not be individually identified, +but you can skip the entire test block if you wish: + +```yaml +lint: + nextflow_config: False +``` + +**The configuration should contain the following or the test will fail:** + +- A `test` configuration profile should exist. + +**The default values in \`\`nextflow.config\`\` should match the default values defined in the \`\`nextflow_schema.json\`\`.** + +:::note +You can choose to ignore tests for the default value of an specific parameter +by creating a file called `.nf-core.yml` in the root of your pipeline and creating +a list the config parameters that should be ignored. For example to ignore the default value for the input parameter: + +```yaml +lint: + nextflow_config: + - config_defaults: + - params.input +``` + +::: diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/pipeline_name_conventions.md b/src/content/tools/docs/dev/pipeline_lint_tests/pipeline_name_conventions.md new file mode 100644 index 0000000000..788d77e41a --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/pipeline_name_conventions.md @@ -0,0 +1,13 @@ +# pipeline_name_conventions + +#### `PipelineLint.pipeline_name_conventions(){:python}` + +Checks that the pipeline name adheres to nf-core conventions. + +In order to ensure consistent naming, pipeline names should contain only lower case, alphanumeric characters. +Otherwise a warning is displayed. + +:::warning +DockerHub is very picky about image names and doesn’t even allow hyphens (we are `nfcore`). +This is a large part of why we set this rule. +::: diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/pipeline_todos.md b/src/content/tools/docs/dev/pipeline_lint_tests/pipeline_todos.md new file mode 100644 index 0000000000..ea401161ac --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/pipeline_todos.md @@ -0,0 +1,28 @@ +# pipeline_todos + +#### `PipelineLint.pipeline_todos(root_dir=None){:python}` + +Check for nf-core _TODO_ lines. + +The nf-core workflow template contains a number of comment lines to help developers +of new pipelines know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the pipeline and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/readme.md b/src/content/tools/docs/dev/pipeline_lint_tests/readme.md new file mode 100644 index 0000000000..a0c30a62be --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/readme.md @@ -0,0 +1,22 @@ +# readme + +#### `PipelineLint.readme(){:python}` + +Repository `README.md` tests + +The `README.md` files for a project are very important and must meet some requirements: + +- Nextflow badge + - If no Nextflow badge is found, a warning is given + - If a badge is found but the version doesn’t match the minimum version in the config file, the test fails + - Example badge code: + ```md + [![Nextflow](https://img.shields.io/badge/nextflow-%E2%89%A50.27.6-brightgreen.svg)](https://www.nextflow.io/) + ``` + +:::note +This badge are a markdown image `![alt-text]()` _inside_ a markdown link `[markdown image]()`, so a bit fiddly to write. +::: + +- Zenodo release + > - If pipeline is released but still contains a ‘zenodo.XXXXXXX’ tag, the test fails diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/schema_description.md b/src/content/tools/docs/dev/pipeline_lint_tests/schema_description.md new file mode 100644 index 0000000000..762ad42ebd --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/schema_description.md @@ -0,0 +1,12 @@ +# schema_description + +#### `PipelineLint.schema_description(){:python}` + +Check that every parameter in the schema has a description. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter. + +Furthermore warns about parameters outside of groups. + +- Warning: Parameters in `nextflow_schema.json` without a description +- Warning: Parameters in `nextflow_schema.json` that are defined outside of a group diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/schema_lint.md b/src/content/tools/docs/dev/pipeline_lint_tests/schema_lint.md new file mode 100644 index 0000000000..d03f77fdde --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/schema_lint.md @@ -0,0 +1,60 @@ +# schema_lint + +#### `PipelineLint.schema_lint(){:python}` + +Pipeline schema syntax + +Pipelines should have a `nextflow_schema.json` file that describes the different +pipeline parameters (eg. `params.something`, `--something`). + +:::note +Reminder: you should generally never need to edit this JSON file by hand. +The `nf-core schema build` command can create _and edit_ the file for you +to keep it up to date, with a friendly user-interface for customisation. +::: + +The lint test checks the schema for the following: + +- Schema should be a valid JSON file +- Schema should adhere to [JSONSchema](https://json-schema.org/), Draft 7. +- Parameters can be described in two places: + > - As `properties` in the top-level schema object + > - As `properties` within subschemas listed in a top-level `definitions` objects +- The schema must describe at least one parameter +- There must be no duplicate parameter IDs across the schema and definition subschema +- All subschema in `definitions` must be referenced in the top-level `allOf` key +- The top-level `allOf` key must not describe any non-existent definitions +- Default parameters in the schema must be valid +- Core top-level schema attributes should exist and be set as follows: + > - `$schema`: `https://json-schema.org/draft-07/schema` + > - `$id`: URL to the raw schema file, eg. `https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json` + > - `title`: `YOURPIPELINE pipeline parameters` + > - `description`: The pipeline config `manifest.description` +- That the `input` property is defined and has a mimetype. A list of common mimetypes can be found [here](https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types). + +For example, an _extremely_ minimal schema could look like this: + +```json +{ + "$schema": "https://json-schema.org/draft-07/schema", + "$id": "https://raw.githubusercontent.com/YOURPIPELINE/master/nextflow_schema.json", + "title": "YOURPIPELINE pipeline parameters", + "description": "This pipeline is for testing", + "properties": { + "first_param": { "type": "string" } + }, + "definitions": { + "my_first_group": { + "properties": { + "second_param": { "type": "string" } + } + } + }, + "allOf": [{ "$ref": "#/definitions/my_first_group" }] +} +``` + +:::note +You can check your pipeline schema without having to run the entire pipeline lint +by running `nf-core schema lint` instead of `nf-core lint` +::: diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/schema_params.md b/src/content/tools/docs/dev/pipeline_lint_tests/schema_params.md new file mode 100644 index 0000000000..7d9440fa15 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/schema_params.md @@ -0,0 +1,11 @@ +# schema_params + +#### `PipelineLint.schema_params(){:python}` + +Check that the schema describes all flat params in the pipeline. + +The `nextflow_schema.json` pipeline schema should describe every flat parameter +returned from the `nextflow config` command (params that are objects or more complex structures are ignored). + +- Failure: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` +- Warning: If parameters are found in `nextflow_schema.json` that are not in `nextflow_schema.json` diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/system_exit.md b/src/content/tools/docs/dev/pipeline_lint_tests/system_exit.md new file mode 100644 index 0000000000..dc54311edc --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/system_exit.md @@ -0,0 +1,10 @@ +# system_exit + +#### `PipelineLint.system_exit(){:python}` + +Check for System.exit calls in groovy/nextflow code + +Calls to System.exit(1) should be replaced by throwing errors + +This lint test looks for all calls to System.exit +in any file with the .nf or .groovy extension diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/template_strings.md b/src/content/tools/docs/dev/pipeline_lint_tests/template_strings.md new file mode 100644 index 0000000000..37b9794794 --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/template_strings.md @@ -0,0 +1,37 @@ +# template_strings + +#### `PipelineLint.template_strings(){:python}` + +Check for template placeholders. + +The `nf-core create` pipeline template uses +[Jinja](https://jinja.palletsprojects.com/en/2.11.x/) behind the scenes. + +This lint test fails if any Jinja template variables such as +`{{ pipeline_name }}` are found in your pipeline code. + +Finding a placeholder like this means that something was probably copied and pasted +from the template without being properly rendered for your pipeline. + +This test ignores any double-brackets prefixed with a dollar sign, such as +`${{ secrets.AWS_ACCESS_KEY_ID }}` as these placeholders are used in GitHub Actions workflows. + +:::note +You can choose to ignore lint test tests by editing the file called +`.nf-core.yml` in the root of your pipeline and setting the test to false: + +```yaml +lint: + template_strings: False +``` + +::: + +To disable this test only for specific files, you can specify a list of file paths to ignore. +For example, to ignore a pdf you added to the docs: + +```yaml +lint: + template_strings: + - docs/my_pdf.pdf +``` diff --git a/src/content/tools/docs/dev/pipeline_lint_tests/version_consistency.md b/src/content/tools/docs/dev/pipeline_lint_tests/version_consistency.md new file mode 100644 index 0000000000..040f5e57fc --- /dev/null +++ b/src/content/tools/docs/dev/pipeline_lint_tests/version_consistency.md @@ -0,0 +1,23 @@ +# version_consistency + +#### `PipelineLint.version_consistency(){:python}` + +Pipeline and container version number consistency. + +:::note +This test only runs when the `--release` flag is set for `nf-core lint`, +or `$GITHUB_REF` is equal to `master`. +::: + +This lint fetches the pipeline version number from three possible locations: + +- The pipeline config, `manifest.version` +- The docker container in the pipeline config, `process.container` + > - Some pipelines may not have this set on a pipeline level. If it is not found, it is ignored. +- `$GITHUB_REF`, if it looks like a release tag (`refs/tags/`) + +The test then checks that: + +- The container name has a tag specified (eg. `nfcore/pipeline:version`) +- The pipeline version number is numeric (contains only numbers and dots) +- That the version numbers all match one another diff --git a/src/content/tools/docs/dev/subworkflow_lint_tests/index.md b/src/content/tools/docs/dev/subworkflow_lint_tests/index.md new file mode 100644 index 0000000000..d5061028dc --- /dev/null +++ b/src/content/tools/docs/dev/subworkflow_lint_tests/index.md @@ -0,0 +1,16 @@ +# Subworkflow lint tests + +# Tests: + +- [main_nf](main_nf) + - [`SubworkflowLint.main_nf()`](main_nf#nf_core.subworkflows.lint.SubworkflowLint.main_nf) +- [meta_yml](meta_yml) + - [`SubworkflowLint.meta_yml()`](meta_yml#nf_core.subworkflows.lint.SubworkflowLint.meta_yml) +- [subworkflow_changes](subworkflow_changes) + - [`SubworkflowLint.subworkflow_changes()`](subworkflow_changes#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_changes) +- [subworkflow_tests](subworkflow_tests) + - [`SubworkflowLint.subworkflow_tests()`](subworkflow_tests#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_tests) +- [subworkflow_todos](subworkflow_todos) + - [`SubworkflowLint.subworkflow_todos()`](subworkflow_todos#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_todos) +- [subworkflow_version](subworkflow_version) + - [`SubworkflowLint.subworkflow_version()`](subworkflow_version#nf_core.subworkflows.lint.SubworkflowLint.subworkflow_version) diff --git a/src/content/tools/docs/dev/subworkflow_lint_tests/main_nf.md b/src/content/tools/docs/dev/subworkflow_lint_tests/main_nf.md new file mode 100644 index 0000000000..1280380868 --- /dev/null +++ b/src/content/tools/docs/dev/subworkflow_lint_tests/main_nf.md @@ -0,0 +1,16 @@ +# main_nf + +#### `SubworkflowLint.main_nf(subworkflow){:python}` + +Lint a `main.nf` subworkflow file + +Can also be used to lint local subworkflow files, +in which case failures will be reported as +warnings. + +The test checks for the following: + +- A subworkflow SHOULD import at least two modules +- All included modules or subworkflows are used and their names are used for versions.yml +- The workflow name is all capital letters +- The subworkflow emits a software version diff --git a/src/content/tools/docs/dev/subworkflow_lint_tests/meta_yml.md b/src/content/tools/docs/dev/subworkflow_lint_tests/meta_yml.md new file mode 100644 index 0000000000..2ab290d54b --- /dev/null +++ b/src/content/tools/docs/dev/subworkflow_lint_tests/meta_yml.md @@ -0,0 +1,17 @@ +# meta_yml + +#### `SubworkflowLint.meta_yml(subworkflow){:python}` + +Lint a `meta.yml` file + +The lint test checks that the subworkflow has +a `meta.yml` file and that it follows the +JSON schema defined in the `subworkflows/yaml-schema.json` +file in the nf-core/modules repository. + +In addition it checks that the subworkflow name +and subworkflow input is consistent between the +`meta.yml` and the `main.nf`. + +Checks that all input and output channels are specified in `meta.yml`. +Checks that all included components in `main.nf` are specified in `meta.yml`. diff --git a/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_changes.md b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_changes.md new file mode 100644 index 0000000000..cda417f1b6 --- /dev/null +++ b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_changes.md @@ -0,0 +1,14 @@ +# subworkflow_changes + +#### `SubworkflowLint.subworkflow_changes(subworkflow){:python}` + +Checks whether installed nf-core subworkflow have changed compared to the +original repository + +Downloads the `main.nf` and `meta.yml` files for every subworkflow +and compares them to the local copies + +If the subworkflow has a commit SHA entry in the `modules.json`, the file content is +compared against the files in the remote at this SHA. + +Only runs when linting a pipeline, not the modules repository diff --git a/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_tests.md b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_tests.md new file mode 100644 index 0000000000..240405567b --- /dev/null +++ b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_tests.md @@ -0,0 +1,10 @@ +# subworkflow_tests + +#### `SubworkflowLint.subworkflow_tests(subworkflow: NFCoreComponent){:python}` + +Lint the tests of a subworkflow in `nf-core/modules` + +It verifies that the test directory exists +and contains a `main.nf.test` a `main.nf.test.snap` and `tags.yml`. + +Additionally, hecks that all included components in test `main.nf` are specified in `test.yml` diff --git a/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_todos.md b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_todos.md new file mode 100644 index 0000000000..e17f92c01d --- /dev/null +++ b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_todos.md @@ -0,0 +1,28 @@ +# subworkflow_todos + +#### `SubworkflowLint.subworkflow_todos(subworkflow){:python}` + +Look for TODO statements in the subworkflow files + +The nf-core subworkflow template contains a number of comment lines to help developers +of new subworkflow know where they need to edit files and add content. +They typically have the following format: + +```groovy +// TODO nf-core: Make some kind of change to the workflow here +``` + +..or in markdown: + +```html + +``` + +This lint test runs through all files in the subworkflows and searches for these lines. +If any are found they will throw a warning. + +:::note +Note that many GUI code editors have plugins to list all instances of _TODO_ +in a given project directory. This is a very quick and convenient way to get +started on your pipeline! +::: diff --git a/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_version.md b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_version.md new file mode 100644 index 0000000000..9b398909f7 --- /dev/null +++ b/src/content/tools/docs/dev/subworkflow_lint_tests/subworkflow_version.md @@ -0,0 +1,9 @@ +# subworkflow_version + +#### `SubworkflowLint.subworkflow_version(subworkflow){:python}` + +Verifies that the subworkflow has a version specified in the `modules.json` file + +It checks whether the subworkflow has an entry in the `modules.json` file +containing a commit SHA. If that is true, it verifies that there are no +newer version of the subworkflow available. diff --git a/src/layouts/MarkdownTocLayout.astro b/src/layouts/MarkdownTocLayout.astro index b1ee46d712..1d314098e0 100644 --- a/src/layouts/MarkdownTocLayout.astro +++ b/src/layouts/MarkdownTocLayout.astro @@ -101,13 +101,15 @@ const {
      { leftSidebar && ( -
      - {sidebarTitle && ( -
      - {sidebarTitle} -
      - )} - +
      +
      + {sidebarTitle && ( +
      + {sidebarTitle} +
      + )} + +
      ) } @@ -121,6 +123,7 @@ const { { headings && headings.length > 1 && (
      + diff --git a/src/pages/tools/docs/[version]/[...slug].astro b/src/pages/tools/docs/[version]/[...slug].astro new file mode 100644 index 0000000000..6b2aa8712b --- /dev/null +++ b/src/pages/tools/docs/[version]/[...slug].astro @@ -0,0 +1,188 @@ +--- +import MarkdownTocLayout from '@layouts/MarkdownTocLayout.astro'; +import { getCollection } from 'astro:content'; +import type { CollectionEntry } from 'astro:content'; +import { sequence } from 'astro:middleware'; +// 1. Generate a new path for every collection entry +export async function getStaticPaths() { + const tools: CollectionEntry<'tools'>[] = await getCollection('tools'); + const versions = [...new Set(tools.map((entry) => entry.id.split('/')[1]))].sort((a, b) => { + const a_version = a.split('.').map((entry) => parseInt(entry)); + const b_version = b.split('.').map((entry) => parseInt(entry)); + for (let i = 0; i < a_version.length; i++) { + if (a_version[i] > b_version[i]) { + return -1; + } + if (a_version[i] < b_version[i]) { + return 1; + } + } + return 0; + }); + // put dev version first + versions.sort((a, b) => { + if (a === 'dev') { + return -1; + } + if (b === 'dev') { + return 1; + } + return 0; + }); + let tools_path = tools.map((entry: CollectionEntry<'tools'>) => { + const version = entry.id.split('/')[1]; + const versioned_tools = tools.filter((tool) => tool.id.split('/')[1] === version); + const section_names = [...new Set(versioned_tools.map((entry) => entry.id.split('/')[2]))]; + const sections = section_names + .filter((entry) => entry !== 'index.md') + .map((entry) => { + return { + section_name: entry, + section_slug: + '/tools/' + + versioned_tools + .find( + (tool_entry) => + tool_entry.id.endsWith('index.md') && tool_entry.id.split('/')[2] === entry, + ) + ?.id.replace(/index\.md$/, 'index'), + children: versioned_tools + .filter((child) => child.id.split('/')[2] === entry && child.id.split('/')[3]) + .filter((child) => child.id.split('/')[3] !== 'index.md') + .map((tool_entry) => { + return { + title: `${tool_entry.id.split('/')[3].replace(/\.md$/, '')}`, + slug: '/tools/' + tool_entry.id.replace(/\.md$/, ''), + }; + }), + }; + }); + // reshuffle sections with section name pipeline_lint_tests first, then modules_lint_tests, then subworkflows_lint_tests and then api + sections.sort((a, b) => { + if (a.section_name === 'pipeline_lint_tests') { + return -1; + } + if (b.section_name === 'pipeline_lint_tests') { + return 1; + } + if (a.section_name === 'api') { + return -1; + } + if (b.section_name === 'api') { + return -1; + } + return 0; + }); + + return { + params: { version: version, slug: entry.id.replace(`docs/${version}/`, '').replace(/\.md$/, '') }, + props: { entry: entry, sections: sections, versions: versions }, + }; + }); + // add 'latest' as a version, which is a copy of the latest version + const latest_version = tools_path.filter((entry) => entry.params.version === versions[1]); + latest_version.map((entry) => { + tools_path.push({ + params: { version: 'latest', slug: entry.params.slug }, + props: { entry: entry.props.entry, sections: entry.props.sections, versions: versions }, + }); + }); + return tools_path; +} +// 2. For your template, you can get the entry directly from the prop +const { entry, sections, versions } = Astro.props; +const { version } = Astro.params; +let { headings, Content } = await entry.render(); +const title = 'nf-core/tools API documentation'; +const version_tag_name = version === 'latest' ? versions[1] : version; +const subtitle = + 'Automated documentation for nf-core/tools ' + + (version_tag_name === 'dev' ? '(`dev` version)' : '(`' + version_tag_name + '`)'); + +const module_name = headings[0].text; +headings.map((heading) => { + // remove everything inside paranthesis + heading.text = heading.text.replace(/\(.*\)/, '()'); + // remove module name from headings + heading.text = heading.text.replace(module_name + '.', ''); + // remove class in the beginning of the heading + heading.text = heading.text.replace(/^class/, ''); + // add word-break hints after before periods + heading.text = heading.text.replaceAll(/\./g, '.\u200B'); +}); + +// add top section wih link to the top index if it doesn't exist +if (!sections.find((section) => section.section_name === 'Contents')) { + sections.unshift({ + section_name: 'Contents', + section_slug: '/tools/docs/' + version + '/index', + children: [], + }); +} + +// try to figure out github URL to the source file from file path +let md_github_url = ''; +const base_url = `https://github.com/nf-core/tools/blob/${version_tag_name}/nf_core/`; +const file_name = entry.id?.split('/').pop()?.replace('.md', '.py') || ''; +// check if entry.id contains "pipeline_lint_tests" +if (entry.id.includes('pipeline_lint_tests')) { + md_github_url = base_url + 'lint/' + file_name; +} else if (entry.id.includes('module_lint_tests') || entry.id.includes('subworkflow_lint_tests')) { + // append to base url either to modules/lint or subworkflows/lint + const component_type = entry.id.includes('module_lint_tests') ? 'modules' : 'subworkflows'; + md_github_url = base_url + component_type + '/lint/' + file_name; +} else if (entry.id.includes('api')) { + md_github_url = base_url + file_name; +} +--- + + +
      + +
      + + +
        + { + versions && + versions.map((v) => { + let url = Astro.url.pathname.replace(`tools/docs/${version}`, `tools/docs/${v}`); + return ( +
      • + + {v} + +
        • + ); + }) + } +
      +
      +
      + +       + diff --git a/src/pages/tools/docs/index.astro b/src/pages/tools/docs/index.astro new file mode 100644 index 0000000000..a609bf623c --- /dev/null +++ b/src/pages/tools/docs/index.astro @@ -0,0 +1,51 @@ +--- +import PageLayout from '@layouts/PageLayout.astro'; +import SidebarNav from '@components/sidebar/SidebarNav.astro'; +import { getCollection } from 'astro:content'; +const files = await getCollection('tools'); +let versions = [...new Set(files.map((file) => file.id.split('/')[1]))]; +// sort semver +versions = versions.sort((a, b) => { + const a_split = a.split('.'); + const b_split = b.split('.'); + for (let i = 0; i < 3; i++) { + if (parseInt(a_split[i]) > parseInt(b_split[i])) { + return -1; + } else if (parseInt(a_split[i]) < parseInt(b_split[i])) { + return 1; + } + } + return 0; +}); + +// move dev to the start +versions = versions.filter((v) => v !== 'dev'); +versions.unshift('dev'); +const md_github_url = 'https://github.com/nf-core/tools/blob/master/README.md'; +--- + + +
      +
      + +
      + +
      +
      +
      +

      nfcore/tools releases

      +
        + { + versions && + versions.map((version) => ( +
      • + {version} +
        • + )) + } +
      +
      +
      +
      +
      +