From 549f38ceec26a17d16b363e216b30972b68cfdab Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Fri, 2 Dec 2022 13:32:34 -0800 Subject: [PATCH 01/11] Request build on linux-gpu --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8a19254..792ed9a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: build-and-test: strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest, macos-latest, linux-gpu] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 From 0364ad13c76140f1ee0d1f83488e80b06c4f268a Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Fri, 2 Dec 2022 13:35:41 -0800 Subject: [PATCH 02/11] Explicitly request miniconda --- .github/workflows/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 792ed9a..bcc3da7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,6 +15,7 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, linux-gpu] runs-on: ${{ matrix.os }} + miniconda-version: "latest" steps: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2 From 34d46b33b1fc2507ffaa9f2db6bbbfffef4a65a5 Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Fri, 2 Dec 2022 13:40:17 -0800 Subject: [PATCH 03/11] Explicitly request minicondaush --- .github/workflows/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index bcc3da7..32a94a3 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,7 +19,8 @@ jobs: steps: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2 - with: + with: + miniconda-version: "latest" auto-update-conda: true - name: Install shell: bash -l {0} From 5cb71f5aae284be02778053c83a0b9409664d01a Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Fri, 2 Dec 2022 13:45:45 -0800 Subject: [PATCH 04/11] Fix typo --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 32a94a3..d42ff38 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -15,7 +15,6 @@ jobs: matrix: os: [ubuntu-latest, macos-latest, linux-gpu] runs-on: ${{ matrix.os }} - miniconda-version: "latest" steps: - uses: actions/checkout@v2 - uses: conda-incubator/setup-miniconda@v2 From 376a1947a02d52c75e7a51096bde34919bbab76a Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Fri, 2 Dec 2022 15:11:13 -0800 Subject: [PATCH 05/11] Preserve stop_idx to avoid race conditions on descruction --- src/unifrac_task.hpp | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/unifrac_task.hpp b/src/unifrac_task.hpp index 091320f..64a5fef 100644 --- a/src/unifrac_task.hpp +++ b/src/unifrac_task.hpp @@ -53,22 +53,23 @@ namespace SUCMP_NM { public: const unsigned int start_idx; + const unsigned int stop_idx; const unsigned int n_samples; const uint64_t n_samples_r; TFloat* const buf; UnifracTaskVector(std::vector &_dm_stripes, const su::task_parameters* _task_p) : dm_stripes(_dm_stripes), task_p(_task_p) - , start_idx(task_p->start), n_samples(task_p->n_samples) + , start_idx(task_p->start), stop_idx(task_p->stop), n_samples(task_p->n_samples) , n_samples_r(((n_samples + UNIFRAC_BLOCK-1)/UNIFRAC_BLOCK)*UNIFRAC_BLOCK) // round up - , buf((dm_stripes[start_idx]==NULL) ? NULL : new TFloat[n_samples_r*(task_p->stop-start_idx)]) // dm_stripes could be null, in which case keep it null + , buf((dm_stripes[start_idx]==NULL) ? NULL : new TFloat[n_samples_r*(stop_idx-start_idx)]) // dm_stripes could be null, in which case keep it null { TFloat* const ibuf = buf; if (ibuf != NULL) { #ifdef _OPENACC - const uint64_t bufels = n_samples_r * (task_p->stop-start_idx); + const uint64_t bufels = n_samples_r * (stop_idx-start_idx); #endif - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { + for(unsigned int stripe=start_idx; stripe < stop_idx; stripe++) { double * dm_stripe = dm_stripes[stripe]; TFloat * buf_stripe = this->operator[](stripe); for(unsigned int j=0; jstop-start_idx); + const uint64_t bufels = n_samples_r * (stop_idx-start_idx); #pragma acc exit data copyout(ibuf[:bufels]) #endif - for(unsigned int stripe=start_idx; stripe < task_p->stop; stripe++) { + for(unsigned int stripe=start_idx; stripe < stop_idx; stripe++) { double * dm_stripe = dm_stripes[stripe]; TFloat * buf_stripe = this->operator[](stripe); for(unsigned int j=0; j Date: Mon, 5 Dec 2022 11:26:59 -0800 Subject: [PATCH 06/11] Change target to linux-gpu-cuda --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d42ff38..76d72fe 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -13,7 +13,7 @@ jobs: build-and-test: strategy: matrix: - os: [ubuntu-latest, macos-latest, linux-gpu] + os: [ubuntu-latest, macos-latest, linux-gpu-cuda] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v2 From d30cf3a3d32d55974a46b3f2aa505fadb59c274c Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Tue, 6 Dec 2022 08:45:45 -0800 Subject: [PATCH 07/11] Explicitly delete copy constructor and assignment --- src/unifrac_task.hpp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/src/unifrac_task.hpp b/src/unifrac_task.hpp index 64a5fef..7900059 100644 --- a/src/unifrac_task.hpp +++ b/src/unifrac_task.hpp @@ -87,6 +87,9 @@ namespace SUCMP_NM { } } + UnifracTaskVector(const UnifracTaskVector& ) = delete; + UnifracTaskVector& operator= (const UnifracTaskVector&) = delete; + TFloat * operator[](unsigned int idx) { return buf+((idx-start_idx)*n_samples_r);} const TFloat * operator[](unsigned int idx) const { return buf+((idx-start_idx)*n_samples_r);} @@ -148,6 +151,9 @@ namespace SUCMP_NM { : dm_stripes(baseObj.dm_stripes), dm_stripes_total(baseObj.dm_stripes_total), task_p(baseObj.task_p) {} */ + UnifracTaskBase(const UnifracTaskBase& ) = delete; + UnifracTaskBase& operator= (const UnifracTaskBase&) = delete; + virtual ~UnifracTaskBase() { #ifdef _OPENACC @@ -317,6 +323,8 @@ namespace SUCMP_NM { , embedded_proportions(_embedded_proportions), max_embs(_max_embs) {} */ + UnifracTask(const UnifracTask& ) = delete; + UnifracTask& operator= (const UnifracTask&) = delete; virtual ~UnifracTask() {} @@ -347,6 +355,9 @@ namespace SUCMP_NM { #pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) } + UnifracUnnormalizedWeightedTask(const UnifracUnnormalizedWeightedTask& ) = delete; + UnifracUnnormalizedWeightedTask& operator= (const UnifracUnnormalizedWeightedTask&) = delete; + virtual ~UnifracUnnormalizedWeightedTask() { #ifdef _OPENACC @@ -382,6 +393,9 @@ namespace SUCMP_NM { #pragma acc enter data create(zcheck[:n_samples],sums[:n_samples]) } + UnifracNormalizedWeightedTask(const UnifracNormalizedWeightedTask& ) = delete; + UnifracNormalizedWeightedTask& operator= (const UnifracNormalizedWeightedTask&) = delete; + virtual ~UnifracNormalizedWeightedTask() { #ifdef _OPENACC @@ -422,6 +436,9 @@ namespace SUCMP_NM { #pragma acc enter data create(zcheck[:n_samples],stripe_sums[:n_samples],sums[:bsize]) } + UnifracUnweightedTask(const UnifracUnweightedTask& ) = delete; + UnifracUnweightedTask& operator= (const UnifracUnweightedTask&) = delete; + virtual ~UnifracUnweightedTask() { #ifdef _OPENACC @@ -451,6 +468,9 @@ namespace SUCMP_NM { UnifracGeneralizedTask(std::vector &_dm_stripes, std::vector &_dm_stripes_total, unsigned int _max_embs, const su::task_parameters* _task_p) : UnifracTask(_dm_stripes,_dm_stripes_total,_max_embs,_task_p) {} + UnifracGeneralizedTask(const UnifracGeneralizedTask& ) = delete; + UnifracGeneralizedTask& operator= (const UnifracGeneralizedTask&) = delete; + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} void _run(unsigned int filled_embs, const TFloat * __restrict__ length); @@ -517,6 +537,8 @@ namespace SUCMP_NM { , embedded_proportions(_embedded_proportions), embedded_counts(initialize_embedded()), sample_total_counts(_sample_total_counts), max_embs(_max_embs) {} */ + UnifracVawTask(const UnifracVawTask& ) = delete; + UnifracVawTask& operator= (const UnifracVawTask&) = delete; virtual ~UnifracVawTask() { @@ -555,6 +577,9 @@ namespace SUCMP_NM { unsigned int _max_embs, const su::task_parameters* _task_p) : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + UnifracVawUnnormalizedWeightedTask(const UnifracVawUnnormalizedWeightedTask& ) = delete; + UnifracVawUnnormalizedWeightedTask& operator= (const UnifracVawUnnormalizedWeightedTask&) = delete; + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} void _run(unsigned int filled_embs, const TFloat * __restrict__ length); @@ -567,6 +592,9 @@ namespace SUCMP_NM { unsigned int _max_embs, const su::task_parameters* _task_p) : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + UnifracVawNormalizedWeightedTask(const UnifracVawNormalizedWeightedTask& ) = delete; + UnifracVawNormalizedWeightedTask& operator= (const UnifracVawNormalizedWeightedTask&) = delete; + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} void _run(unsigned int filled_embs, const TFloat * __restrict__ length); @@ -579,6 +607,9 @@ namespace SUCMP_NM { unsigned int _max_embs, const su::task_parameters* _task_p) : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + UnifracVawUnweightedTask(const UnifracVawUnweightedTask& ) = delete; + UnifracVawUnweightedTask& operator= (const UnifracVawUnweightedTask&) = delete; + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} void _run(unsigned int filled_embs, const TFloat * __restrict__ length); @@ -591,6 +622,9 @@ namespace SUCMP_NM { unsigned int _max_embs, const su::task_parameters* _task_p) : UnifracVawTask(_dm_stripes,_dm_stripes_total,_sample_total_counts,_max_embs,_task_p) {} + UnifracVawGeneralizedTask(const UnifracVawGeneralizedTask& ) = delete; + UnifracVawGeneralizedTask& operator= (const UnifracVawGeneralizedTask&) = delete; + virtual void run(unsigned int filled_embs, const TFloat * __restrict__ length) {_run(filled_embs, length);} void _run(unsigned int filled_embs, const TFloat * __restrict__ length); From 83b88689df033e7b02f50a2904f6d4ea35a1ae3c Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Tue, 6 Dec 2022 08:46:47 -0800 Subject: [PATCH 08/11] Properly return printf result --- src/unifrac_internal.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/unifrac_internal.cpp b/src/unifrac_internal.cpp index 1fff6e1..85551dc 100644 --- a/src/unifrac_internal.cpp +++ b/src/unifrac_internal.cpp @@ -29,10 +29,11 @@ static int sync_printf(const char *format, ...) { va_start(args, format); pthread_mutex_lock(&printf_mutex); - vprintf(format, args); + int cnt = vprintf(format, args); pthread_mutex_unlock(&printf_mutex); va_end(args); + return cnt; } static void sig_handler(int signo) { From 6cceeeeb8f86a8fee3dda64442866504d80c7540 Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Tue, 6 Dec 2022 09:24:43 -0800 Subject: [PATCH 09/11] Reduce verbosity --- .github/workflows/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 76d72fe..5293ca7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -35,9 +35,9 @@ jobs: if [[ "$(uname -s)" == "Linux" ]]; then - conda create --yes -n unifrac -c conda-forge -c bioconda gxx_linux-64 hdf5 mkl-include lz4 zlib hdf5-static libcblas liblapacke make curl + conda create -q --yes -n unifrac -c conda-forge -c bioconda gxx_linux-64 hdf5 mkl-include lz4 zlib hdf5-static libcblas liblapacke make curl else - conda create --yes -n unifrac -c conda-forge -c bioconda clangxx_osx-64 hdf5 mkl-include lz4 hdf5-static libcblas liblapacke make curl + conda create -q --yes -n unifrac -c conda-forge -c bioconda clangxx_osx-64 hdf5 mkl-include lz4 hdf5-static libcblas liblapacke make curl fi conda activate unifrac echo "$(uname -s)" @@ -53,7 +53,7 @@ jobs: which h5c++ if [[ "$(uname -s)" == "Linux" ]]; then - ./scripts/install_hpc_sdk.sh + ./scripts/install_hpc_sdk.sh >/dev/null 2>/dev/null Date: Tue, 6 Dec 2022 09:27:37 -0800 Subject: [PATCH 10/11] Reduce verbosity in conda build --- src/Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Makefile b/src/Makefile index 3d12c71..b597cf9 100644 --- a/src/Makefile +++ b/src/Makefile @@ -58,9 +58,9 @@ ifndef NOGPU ACCCPPFLAGS += -ta=tesla:ccall else ACCCPPFLAGS += -ta=tesla + # optional info + ACCCPPFLAGS += -Minfo=accel endif - # optional info - ACCCPPFLAGS += -Minfo=accel LDDFLAGS += -shlib -acc -Bstatic_pgi EXEFLAGS += -acc -Bstatic_pgi endif @@ -79,8 +79,10 @@ else endif ifeq (,$(findstring pgi,$(COMPILER))) + ifneq ($(PERFORMING_CONDA_BUILD),True) # basically, not gcc CPPFLAGS += -Wextra -Wno-unused-parameter + endif endif BLASLIB=-llapacke -lcblas From 02f24fcb7ca9d2bae84d5156a43b663cbe8ec369 Mon Sep 17 00:00:00 2001 From: Igor Sfiligoi Date: Wed, 7 Dec 2022 08:10:09 -0800 Subject: [PATCH 11/11] Add disk space debug messages --- .github/workflows/main.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5293ca7..a22e380 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -33,12 +33,15 @@ jobs: sudo mv -f /Library/Developer/CommandLineTools /Library/Developer/CommandLineTools.org fi + df -h . if [[ "$(uname -s)" == "Linux" ]]; then conda create -q --yes -n unifrac -c conda-forge -c bioconda gxx_linux-64 hdf5 mkl-include lz4 zlib hdf5-static libcblas liblapacke make curl else conda create -q --yes -n unifrac -c conda-forge -c bioconda clangxx_osx-64 hdf5 mkl-include lz4 hdf5-static libcblas liblapacke make curl fi + conda clean --yes -t + df -h . conda activate unifrac echo "$(uname -s)" if [[ "$(uname -s)" == "Linux" ]]; @@ -56,11 +59,13 @@ jobs: ./scripts/install_hpc_sdk.sh >/dev/null 2>/dev/null