From c898e1e6a91dd90c3926fef583feb9ee5a04bb03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adam=20Gudy=C5=9B?= Date: Mon, 25 Nov 2024 11:55:17 +0100 Subject: [PATCH] Better progress reporting Co-authored-by: Sebastian Deorowicz --- .github/workflows/deploy.yml | 9 +- .github/workflows/main.yml | 17 +- .github/workflows/self-hosted.yml | 46 +- README.md | 2 +- libs/refresh/compression/lib/file_wrapper.h | 55 +- libs/refresh/logs/lib/progress.h | 154 ++++ .../parallel_queues/lib/parallel-queues.h | 43 +- makefile | 166 ++-- refresh.mk | 739 ++++++++++++++++++ src/ani-entropy.vcxproj | 1 + src/ani-entropy.vcxproj.filters | 3 + src/defs.h | 6 +- src/filter.cpp | 89 ++- src/filter.h | 6 + src/lz_matcher.cpp | 52 +- src/seq_reservoir.cpp | 21 +- 16 files changed, 1195 insertions(+), 214 deletions(-) create mode 100644 libs/refresh/logs/lib/progress.h create mode 100644 refresh.mk diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 48a23c0..781ae70 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -17,6 +17,8 @@ jobs: runs-on: [self-hosted, lz-ani, '${{ matrix.machine }}'] steps: + - name: clean + run: rm -rf ${{ github.workspace }}/* - uses: actions/checkout@v4 with: submodules: recursive @@ -47,10 +49,9 @@ jobs: steps: - name: make run: | - make clean - make -j32 CXX=${{matrix.compiler}} STATIC_LINK=true PLATFORM=${{ matrix.platform }} + gmake -j32 CXX=${{matrix.compiler}} STATIC_LINK=true PLATFORM=${{ matrix.platform }} - name: tar artifacts - run: tar -cvzf lz-ani.tar.gz lz-ani LICENSE + run: tar -cvzf lz-ani.tar.gz LICENSE -C ./bin lz-ani ######################################################################################## @@ -65,7 +66,7 @@ jobs: steps: - name: help - run: ./lz-ani + run: ./bin/lz-ani ######################################################################################## upload: diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 2a85e0b..8138858 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -1,4 +1,4 @@ -name: GitHub Actions CI +name: Build and tests on: push: @@ -15,20 +15,25 @@ jobs: strategy: fail-fast: false matrix: - machine: [ubuntu-latest, macOS-12] + machine: [ubuntu-latest] + gmake_install_command: ['gmake --version'] + include: + - {machine: macOS-13, gmake_install_command: 'brew install make && gmake --version'} runs-on: ['${{ matrix.machine }}'] steps: - uses: actions/checkout@v4 with: submodules: recursive + + - name: install gmake + run: ${{ matrix.gmake_install_command }} - name: make run: | - g++ --version - make -j CXX=g++-12 + gmake -j CXX=g++-12 CC=gcc-12 STATIC_LINK=true - name: tar artifacts - run: tar -cvf lz-ani.tar ./lz-ani ./test + run: tar -cvf lz-ani.tar ./test -C ./bin lz-ani - uses: actions/upload-artifact@v4 with: @@ -42,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - machine: [ubuntu-latest, macOS-12] + machine: [ubuntu-latest, macOS-13] runs-on: ['${{ matrix.machine }}'] steps: diff --git a/.github/workflows/self-hosted.yml b/.github/workflows/self-hosted.yml index 84e8524..da0fe1b 100644 --- a/.github/workflows/self-hosted.yml +++ b/.github/workflows/self-hosted.yml @@ -18,6 +18,8 @@ jobs: runs-on: [self-hosted, lz-ani, '${{ matrix.machine }}'] steps: + - name: clean + run: rm -rf ${{ github.workspace }}/* - uses: actions/checkout@v4 with: submodules: recursive @@ -31,28 +33,24 @@ jobs: fail-fast: false matrix: machine: [x64_linux, x64_mac, arm64_linux, arm64_mac] - compiler: [g++-11, g++-12, g++-13] + compiler: [11, 12, 13] include: - - machine: x64_linux - platform: avx2 - - machine: arm64_linux - platform: arm8 - - machine: x64_mac - platform: avx2 - - machine: arm64_mac - platform: m1 + - {machine: x64_linux, platform: avx2, compiler: 14} + - {machine: x64_linux, platform: avx2} + - {machine: arm64_linux, platform: arm8} + - {machine: x64_mac, platform: avx2} + - {machine: arm64_mac, platform: m1} exclude: - - machine: arm64_linux - compiler: g++-13 + - {machine: arm64_linux, compiler: 13} runs-on: [self-hosted, lz-ani, '${{ matrix.machine }}'] steps: - name: make run: | - make -j32 CXX=${{matrix.compiler}} PLATFORM=${{ matrix.platform }} - cp ./lz-ani ./lz-ani-${{matrix.compiler}} - make clean + gmake -j32 CXX=g++-${{matrix.compiler}} CC=gcc-${{matrix.compiler}} PLATFORM=${{ matrix.platform }} STATIC_LINK=true + cp ./bin/lz-ani ./lz-ani-${{matrix.compiler}} + gmake clean ######################################################################################## help: @@ -62,10 +60,11 @@ jobs: fail-fast: false matrix: machine: [x64_linux, x64_mac, arm64_linux, arm64_mac] - compiler: [g++-11, g++-12, g++-13] + compiler: [11, 12, 13] + include: + - {machine: x64_linux, platform: avx2, compiler: 14} exclude: - - machine: arm64_linux - compiler: g++-13 + - {machine: arm64_linux, compiler: 13} runs-on: [self-hosted, lz-ani, '${{ matrix.machine }}'] @@ -74,9 +73,9 @@ jobs: steps: - name: help - run: | - ${EXEC} - + run: ${EXEC} + - name: version + run: ${EXEC} -version ######################################################################################## vir61: @@ -86,10 +85,11 @@ jobs: fail-fast: false matrix: machine: [x64_linux, x64_mac, arm64_linux, arm64_mac] - compiler: [g++-11, g++-12, g++-13] + compiler: [11, 12, 13] + include: + - {machine: x64_linux, platform: avx2, compiler: 14} exclude: - - machine: arm64_linux - compiler: g++-13 + - {machine: arm64_linux, compiler: 13} runs-on: [self-hosted, lz-ani, '${{ matrix.machine }}'] diff --git a/README.md b/README.md index b547ab1..75bd9f2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![GitHub downloads](https://img.shields.io/github/downloads/refresh-bio/lz-ani/total.svg?style=flag&label=GitHub%20downloads)](https://github.com/refresh-bio/LZ-ANI/releases) [![Bioconda downloads](https://img.shields.io/conda/dn/bioconda/lz-ani.svg?style=flag&label=Bioconda%20downloads)](https://anaconda.org/bioconda/lz-ani) -[![GitHub Actions CI](../../workflows/GitHub%20Actions%20CI/badge.svg)](../../actions/workflows/main.yml) +[![Build and tests](../../workflows/Build%20and%20tests/badge.svg)](../../actions/workflows/main.yml) [![License: GPL v3](https://img.shields.io/badge/License-GPLv3-blue.svg)](https://www.gnu.org/licenses/gpl-3.0) diff --git a/libs/refresh/compression/lib/file_wrapper.h b/libs/refresh/compression/lib/file_wrapper.h index fde390e..673381d 100644 --- a/libs/refresh/compression/lib/file_wrapper.h +++ b/libs/refresh/compression/lib/file_wrapper.h @@ -1,16 +1,11 @@ -// ******************************************************************************************* -// This file is a part of LZ-ANI software distributed under GNU GPL 3 license. -// The homepage of the LZ-ANI project is https://github.com/refresh-bio/LZ-ANI -// -// Copyright(C) 2024-2024, S.Deorowicz, A.Gudys -// -// Version: 1.0.0 -// Date : 2024-05-15 -// ******************************************************************************************* - #ifndef _FILE_WRAPPER_H #define _FILE_WRAPPER_H +// *** History of updates +// *** v. 1.0.1 (2024-03-11) - bug fix (wrong zlib initialization) +// *** v. 1.0.2 (2024-05-01) - bug fix (wrong reading from file) +// *** + #include #include #include @@ -46,9 +41,9 @@ #ifdef REFRESH_STREAM_DECOMPRESSION_ENABLE_ZLIB #ifdef _WIN32 -#include "../libs/zlib-ng/build-vs/zlib.h" +#include #else -#include "../zlib-ng/zlib.h" +#include #endif #endif @@ -150,17 +145,30 @@ namespace refresh { std::string file_name; size_t io_buffer_size; - FILE* file = nullptr; - bool test_extension = true; + FILE* file; + bool test_extension; + + void _open() + { + file = fopen(file_name.c_str(), "rb"); + + if (!file) + return; + + setvbuf(file, nullptr, _IOFBF, io_buffer_size); + buffer_released = true; + } public: stream_in_file(const std::string& file_name, size_t io_buffer_size = 16 << 20, size_t buffer_size = 8 << 20, bool test_extension = true) : stream_in_buffered(buffer_size), - file_name(file_name), - io_buffer_size(io_buffer_size), - test_extension(test_extension) + file_name{ file_name }, + io_buffer_size{ io_buffer_size }, + file{ nullptr }, + test_extension{test_extension} { - open(file_name); +// open(file_name); + _open(); } virtual ~stream_in_file() @@ -175,16 +183,9 @@ namespace refresh if (file) close(); - file = fopen(file_name.c_str(), "rb"); - - if (!file) - return false; - - setvbuf(file, nullptr, _IOFBF, io_buffer_size); - - buffer_released = true; + _open(); - return true; + return file != nullptr; } virtual bool close() diff --git a/libs/refresh/logs/lib/progress.h b/libs/refresh/logs/lib/progress.h new file mode 100644 index 0000000..c50e321 --- /dev/null +++ b/libs/refresh/logs/lib/progress.h @@ -0,0 +1,154 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace refresh +{ + class progress_state + { + enum class type_t {none, number, of_total, percent}; + + type_t type{ type_t::none }; + + uint64_t counter{ 0 }; + uint64_t total{ 0 }; + + std::string prefix; + std::string separator; + std::string suffix; + + int precision{ -1 }; + double mult{ 1 }; + + std::string message; + bool message_checked{ false }; + + std::mutex mtx; + + void adjust_precision() + { + if (total == 0) + { + total = 1; // Should be error message here? + } + + if (precision < 0) + { + if (total <= 100) + precision = 0; + else if (total <= 10000) + precision = 1; + else if (total <= 1000000) + precision = 2; + else + precision = 3; + } + else if (precision > 6) + precision = 6; + + mult = pow(10, precision); + } + + void build_message() + { + std::string s; + char buffer[16]; + + if (type == type_t::number) + s = std::to_string(counter); + else if (type == type_t::of_total) + s = prefix + std::to_string(counter) + separator + std::to_string(total) + suffix; + else if (type == type_t::percent) + { + auto r = std::to_chars(buffer, buffer + 16, 100.0 * counter / total, std::chars_format::fixed, precision); + + if (r.ec != std::errc()) + s = ""; + else + { + s = prefix; + s.append(buffer, r.ptr - buffer); + s += suffix; + } + } + + if (s != message) + { + message = move(s); + message_checked = false; + } + } + + public: + progress_state() : + type(type_t::number) + {} + + progress_state(uint64_t total, const std::string &prefix, const std::string &separator, const std::string &suffix) : + type(type_t::of_total), + total(total), + prefix(prefix), + separator(separator), + suffix(suffix) + {} + + progress_state(uint64_t total, const std::string& prefix, const std::string& suffix, int precision) : + type(type_t::percent), + total(total), + prefix(prefix), + suffix(suffix), + precision(precision) + { + adjust_precision(); + } + + const std::string& str() + { + message_checked = true; + return message; + } + + std::string str_ts() + { + std::lock_guard lck(mtx); + + message_checked = true; + + return message; + } + + bool increment(uint64_t n) + { + counter += n; + build_message(); + + return !message_checked; + } + + bool increment_ts(uint64_t n) + { + std::lock_guard lck(mtx); + + counter += n; + build_message(); + + return !message_checked; + } + + bool was_checked() const + { + return message_checked; + } + + bool was_checked_ts() + { + std::lock_guard lck(mtx); + + return message_checked; + } + }; +} \ No newline at end of file diff --git a/libs/refresh/parallel_queues/lib/parallel-queues.h b/libs/refresh/parallel_queues/lib/parallel-queues.h index 9e250da..c6b0f9e 100644 --- a/libs/refresh/parallel_queues/lib/parallel-queues.h +++ b/libs/refresh/parallel_queues/lib/parallel-queues.h @@ -90,7 +90,7 @@ namespace refresh { q.emplace(std::move(elem)); if (was_empty) - cv_pop.notify_one(); + cv_pop.notify_all(); // !!! in some circumstances notify_one may cause threads waiting on pop to never wake up until mark_completed, so decrease the level of parallelism if (queue_observer) queue_observer->notify_pushed(); @@ -169,7 +169,7 @@ namespace refresh { q.pop(); if (was_full) - cv_push.notify_one(); + cv_push.notify_all(); // !!! in some circumstances notify_one may cause threads waiting on push to never wake up and to deadlock if (queue_observer) queue_observer->notify_popped(); @@ -187,6 +187,11 @@ namespace refresh { } } + bool check_completed() { + std::lock_guard lck(mtx); + return is_completed; + } + void cancel() { std::lock_guard lck(mtx); @@ -286,6 +291,35 @@ namespace refresh { return true; } + bool pop(T& elem, uint64_t &priority) + { + auto time_start = std::chrono::high_resolution_clock::now(); + + std::unique_lock lck(mtx); + cv_pop.wait(lck, [this] { + return is_completed || (!map_data.empty() && map_data.begin()->first == current_priority); + }); + + if (queue_observer) + queue_observer->notify_wait_on_pop_time(std::chrono::high_resolution_clock::now() - time_start); + + if (is_completed && map_data.empty()) + return false; + + elem = std::move(map_data.begin()->second); + priority = map_data.begin()->first; + map_data.erase(map_data.begin()); + + ++current_priority; + + cv_push.notify_all(); + + if (queue_observer) + queue_observer->notify_popped(); + + return true; + } + //mkokot_TODO: implement Cancel and PushOrCancel void mark_completed() @@ -357,7 +391,8 @@ namespace refresh { q.emplace(std::move(elem)); if (was_empty) - cv_pop.notify_one(); + cv_pop.notify_all(); //I'm not sure if this is needed instead of notify_one, there is only one producer so its unlikely (impossible?) that at the next push the queue is not empty if it was now empty now and there were waiting consumers + //but just for safety let's keep it notify all if (queue_observer) queue_observer->notify_pushed(); @@ -389,7 +424,7 @@ namespace refresh { q.pop(); if (was_full) - cv_push.notify_one(); + cv_push.notify_one(); //should be fine, since there is only one producer if (queue_observer) queue_observer->notify_popped(); diff --git a/makefile b/makefile index 1db60f9..6f68600 100644 --- a/makefile +++ b/makefile @@ -1,125 +1,57 @@ all: lz-ani -LZANI_ROOT_DIR = . -LZANI_MAIN_DIR = src -LZANI_LIBS_DIR = libs -ISAL_DIR = libs/isa-l -ZLIB_DIR = libs/zlib-ng - -INC_DIRS =. libs/mimalloc/include libs/zlib-ng/ libs/isa-l/include -INCLUDE_DIR=$(foreach d, $(INC_DIRS), -I$d) - -MIMALLOC_INLUCDE_DIR = libs/mimalloc/include - -ifdef MSVC # Avoid the MingW/Cygwin sections - uname_S := Windows - uname_M := "x86_64" -else # If uname not available => 'not' - uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') - uname_M := $(shell sh -c 'uname -m 2>/dev/null || echo not') -endif - -NASM_V := $(shell nasm --version 2>/dev/null) - -ifeq ($(PLATFORM), arm8) -$(info *** ARMv8 with NEON extensions ***) - ARCH_FLAGS := -march=armv8-a -DARCH_ARM -else ifeq ($(PLATFORM), m1) -$(info *** Apple M1(or never) with NEON extensions ***) - ARCH_FLAGS := -march=armv8.4-a -DARCH_ARM -else ifeq ($(PLATFORM), sse2) -$(info *** x86-64 with SSE2 extensions ***) - ARCH_FLAGS := -msse2 -m64 -DARCH_X64 -else ifeq ($(PLATFORM), avx) -$(info *** x86-64 with AVX extensions ***) - ARCH_FLAGS := -mavx -m64 -DARCH_X64 -else ifeq ($(PLATFORM), avx2) -$(info *** x86-64 with AVX2 extensions ***) - ARCH_FLAGS := -mavx2 -m64 -DARCH_X64 -else -$(info *** Unspecified platform - use native compilation) - ifeq ($(uname_M),x86_64) - ARCH_FLAGS := -march=native -DARCH_X64 - else - ARCH_FLAGS := -march=native -DARCH_ARM - endif +# *** REFRESH makefile utils +include refresh.mk + +$(call INIT_SUBMODULES) +$(call INIT_GLOBALS) +$(call CHECK_OS_ARCH, $(PLATFORM)) + +# *** Project directories +$(call SET_SRC_OBJ_BIN,src,obj,bin) +3RD_PARTY_DIR := ./libs + +# *** Project configuration +$(call CHECK_NASM) +$(call PROPOSE_ZLIB_NG, $(3RD_PARTY_DIR)/zlib-ng) +$(call PROPOSE_ISAL, $(3RD_PARTY_DIR)/isa-l) +$(call ADD_MIMALLOC, $(3RD_PARTY_DIR)/mimalloc) +$(call CHOOSE_GZIP_DECOMPRESSION) +$(call ADD_REFRESH_LIB, $(3RD_PARTY_DIR)) +$(call SET_STATIC, $(STATIC_LINK)) +$(call SET_C_CPP_STANDARDS, c11, c++20) +$(call SET_GIT_COMMIT) + +$(call SET_FLAGS, $(TYPE)) + +$(call SET_COMPILER_VERSION_ALLOWED, GCC, Linux_x86_64, 10, 20) +$(call SET_COMPILER_VERSION_ALLOWED, GCC, Linux_aarch64, 11, 20) +$(call SET_COMPILER_VERSION_ALLOWED, GCC, Darwin_x86_64, 11, 13) +$(call SET_COMPILER_VERSION_ALLOWED, GCC, Darwin_arm64, 11, 13) + +ifneq ($(MAKECMDGOALS),clean) +$(call CHECK_COMPILER_VERSION) endif -CFLAGS = -fPIC -static -pthread -Wall -O3 -std=c++20 $(ARCH_FLAGS) $(INCLUDE_DIR) -fpermissive -CLINK = -lm - - -ifeq ($(uname_S),Linux) - CLINK+=-fabi-version=6 - CLINK+=-static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive -endif - -ifeq ($(uname_S),Darwin) - CLINK += -lpthread -static-libgcc -endif +# *** Source files and rules +$(eval $(call PREPARE_DEFAULT_COMPILE_RULE,MAIN,.)) -ifeq ($(uname_M),x86_64) - ifdef NASM_V - GZ_LIB:=isa-l.a - gz_target:=isa-l - CFLAGS+=-DREFRESH_USE_IGZIP - else - GZ_LIB:=libz.a - gz_target:=ng_zlib - CFLAGS+=-DREFRESH_USE_ZLIB - endif -else - GZ_LIB:=libz.a - gz_target:=ng_zlib - CFLAGS+=-DREFRESH_USE_ZLIB -endif - -MIMALLOC_OBJ=libs/mimalloc/mimalloc.o - - - -$(MIMALLOC_OBJ): - $(CC) -DMI_MALLOC_OVERRIDE -O3 -DNDEBUG -fPIC -Wall -Wextra -Wno-unknown-pragmas -fvisibility=hidden -Wstrict-prototypes -ftls-model=initial-exec -fno-builtin-malloc -std=gnu11 -c -I libs/mimalloc/include libs/mimalloc/src/static.c -o $(MIMALLOC_OBJ) - -%.o: %.cpp $(gz_target) - $(CXX) $(CFLAGS) -c $< -o $@ - -ng_zlib: - cd $(ZLIB_DIR) && ./configure --zlib-compat && $(MAKE) libz.a - cp $(ZLIB_DIR)/libz.* $(LZANI_LIBS_DIR) - -isa-l: - cd $(ISAL_DIR) && $(MAKE) -f Makefile.unx - cp $(ISAL_DIR)/bin/isa-l.a $(LZANI_LIBS_DIR) - cp $(ISAL_DIR)/bin/libisal.* $(LZANI_LIBS_DIR) - -lz-ani: $(gz_target) \ - $(LZANI_MAIN_DIR)/lz-ani.o \ - $(LZANI_MAIN_DIR)/filter.o \ - $(LZANI_MAIN_DIR)/lz_matcher.o \ - $(LZANI_MAIN_DIR)/parser.o \ - $(LZANI_MAIN_DIR)/seq_reservoir.o \ - $(LZANI_MAIN_DIR)/utils.o \ - $(MIMALLOC_OBJ) - $(CXX) -o $(LZANI_ROOT_DIR)/$@ \ +# *** Targets +lz-ani: $(OUT_BIN_DIR)/lz-ani +$(OUT_BIN_DIR)/lz-ani: $(GZ_TARGET) mimalloc_obj \ + $(OBJ_MAIN) + -mkdir -p $(OUT_BIN_DIR) + $(CXX) -o $@ \ $(MIMALLOC_OBJ) \ - $(LZANI_MAIN_DIR)/lz-ani.o \ - $(LZANI_MAIN_DIR)/filter.o \ - $(LZANI_MAIN_DIR)/lz_matcher.o \ - $(LZANI_MAIN_DIR)/parser.o \ - $(LZANI_MAIN_DIR)/seq_reservoir.o \ - $(LZANI_MAIN_DIR)/utils.o \ - $(LZANI_LIBS_DIR)/$(GZ_LIB) \ - $(CLINK) + $(OBJ_MAIN) \ + $(LIBRARY_FILES) $(LINKER_FLAGS) $(LINKER_DIRS) + +# *** Cleaning +.PHONY: clean init +clean: clean-zlib-ng clean-isa-l clean-mimalloc_obj + -rm -r $(OBJ_DIR) + -rm -r $(OUT_BIN_DIR) +init: + $(call INIT_SUBMODULES) -clean: - -rm $(LZANI_MAIN_DIR)/*.o - -rm $(LZANI_LIBS_DIR)/*.o - -rm $(MIMALLOC_OBJ) - cd $(ZLIB_DIR) && $(MAKE) -f Makefile.in clean - cd $(ISAL_DIR) && $(MAKE) -f Makefile.unx clean - -rm lz-ani - -rm $(LZANI_LIBS_DIR)/libz.* - -rm $(LZANI_LIBS_DIR)/isa-l.* - -rm $(LZANI_LIBS_DIR)/libisal.* diff --git a/refresh.mk b/refresh.mk new file mode 100644 index 0000000..bb31a9a --- /dev/null +++ b/refresh.mk @@ -0,0 +1,739 @@ +### REFRESH group macros - v.1.0.9 (2024-11-25) + +### Macros for initialization +define INIT_GLOBALS + $(info *** Initialization of global values ***) + $(eval INCLUDE_DIRS:=-I.) + $(eval REFRESH_DIR:=) + $(eval LIBRARY_FILES:=) + $(eval LINKER_DIRS:=) + $(eval C_FLAGS:=) + $(eval CPP_FLAGS:=) + $(eval PY_FLAGS:=) + $(eval DEFINE_FLAGS:=) + $(eval LINKER_FLAGS:=) + $(eval CMAKE_OSX_FIX:=) + $(eval COMPILER_ALLOWED:=) + $(eval TYPE?=release) + $(eval PREBUILD_JOBS:=) + $(eval SRC_DIR:=./src) + $(eval OBJ_DIR:=./obj) + $(eval OUT_BIN_DIR:=./bin) + $(eval AR?=ar) + $(eval NASM?=nasm) +endef + +### Macros for 3rd-party libraries registration +# Add zlib-ng +define ADD_ZLIB_NG + $(info *** Adding zlib-ng ***) + $(eval ZLIB_DIR:=$(1)) + $(eval ZLIB_A_DIR:=$(1)/build-g++/zlib-ng) + $(eval ZLIB_A:=$(ZLIB_A_DIR)/libz.a) + $(eval INCLUDE_DIRS+=-I$(ZLIB_DIR)/build-g++ -I$(ZLIB_DIR)/build-g++/zlib-ng) + $(eval LIBRARY_FILES+=$(ZLIB_A)) + $(eval LINKER_DIRS+=-L $(ZLIB_A_DIR)) + $(eval PREBUILD_JOBS+=zlib-ng) + + $(eval zlib-ng: $(ZLIB_A)) + $(eval $(ZLIB_A) : ; \ + cd $(ZLIB_DIR) && cmake $(CMAKE_OSX_FIX) -DCMAKE_CXX_COMPILER=$(CXX) -DCMAKE_C_COMPILER=$(CC) -B build-g++/zlib-ng -S . -DZLIB_COMPAT=ON; cmake --build build-g++/zlib-ng --config Release) +endef + +# Propose zlib-ng (to be considered by CHOOSE_...) +define PROPOSE_ZLIB_NG + $(info *** Proposing zlib-ng ***) + $(eval ZLIB_DIR:=$(1)) + $(eval ZLIB_A_DIR:=$(1)/build-g++/zlib-ng) + $(eval ZLIB_A:=$(ZLIB_A_DIR)/libz.a) + + $(eval zlib-ng: $(ZLIB_A)) + $(eval $(ZLIB_A) : ; \ + cd $(ZLIB_DIR) && cmake $(CMAKE_OSX_FIX) -DCMAKE_CXX_COMPILER=$(CXX) -DCMAKE_C_COMPILER=$(CC) -B build-g++/zlib-ng -S . -DZLIB_COMPAT=ON; cmake --build build-g++/zlib-ng --config Release) +endef + +# Propose isa-l (to be considered by CHOOSE_...) +define PROPOSE_ISAL + $(info *** Proposing isal ***) + $(eval ISAL_DIR:=$(1)) + $(eval ISAL_A_DIR:=$(1)/bin) + $(eval ISAL_A:=$(1)/bin/isa-l.a) + + $(eval isa-l: $(ISAL_A)) + $(eval $(ISAL_A) : ; \ + cd $(ISAL_DIR) && $(MAKE) -f Makefile.unx) +endef + +# Add libdeflate +define ADD_LIBDEFLATE + $(info *** Adding libdeflate ***) + $(eval INCLUDE_DIRS+=-I$(1)) + $(eval LIBDEFLATE_DIR:=$(1)) + $(eval LIBDEFLATE_A_DIR:=$(1)) + $(eval LIBDEFLATE_A:=$(1)/build/libdeflate.a) + $(eval LIBRARY_FILES+=$(LIBDEFLATE_A)) + $(eval LINKER_DIRS+=-L $(LIBDEFLATE_A_DIR)) + $(call TEST_SOFT,cmake) + $(eval PREBUILD_JOBS+=libdeflate) + + $(eval libdeflate: $(LIBDEFLATE_A)) + $(eval $(LIBDEFLATE_A): ; \ + cd $(LIBDEFLATE_DIR) && cmake $(CMAKE_OSX_FIX) -DCMAKE_CXX_COMPILER=$(CXX) -DCMAKE_C_COMPILER=$(CC) -DLIBDEFLATE_BUILD_SHARED_LIB=OFF -DLIBDEFLATE_BUILD_GZIP=OFF -B build && cmake --build build) +endef + +# Add zstd +define ADD_LIBZSTD + $(info *** Adding libzstd ***) + $(eval INCLUDE_DIRS+=-I$(1)) + $(eval LIBZSTD_DIR:=$(1)) + $(eval LIBZSTD_A_DIR:=$(1)) + $(eval LIBZSTD_A:=$(1)/lib/libzstd.a) + $(eval LIBRARY_FILES+=$(LIBZSTD_A)) + $(eval LINKER_DIRS+=-L $(LIBZSTD_A_DIR)) + $(eval PREBUILD_JOBS+=libzstd) + + $(eval libzstd: $(LIBZSTD_A)) + $(eval $(LIBZSTD_A): ; \ + cd $(LIBZSTD_DIR) && $(MAKE)) +endef + +# Add mimalloc +define ADD_MIMALLOC + $(info *** Adding mimalloc ***) + $(eval MIMALLOC_INCLUDE_DIR:=$(1)/include) + $(eval INCLUDE_DIRS+=-I$(1)/include) + $(eval MIMALLOC_DIR:=$(1)) + $(eval MIMALLOC_OBJ:=$(1)/mimalloc.o) + $(eval PREBUILD_JOBS+=mimalloc_obj) + + $(eval mimalloc_obj: $(MIMALLOC_OBJ)) + $(eval $(MIMALLOC_OBJ): ; \ + $(CXX) -DMI_MALLOC_OVERRIDE -O3 -DNDEBUG -fPIC -Wall -Wextra -Wno-unknown-pragmas \ + -fvisibility=hidden -ftls-model=initial-exec -fno-builtin-malloc -c -I $(MIMALLOC_INCLUDE_DIR) \ + $(MIMALLOC_DIR)/src/static.c -o $(MIMALLOC_OBJ)) +endef + +# Add cdflib +define ADD_CDFLIB + $(info *** Adding cdflib ***) + $(eval CDFLIB_INCLUDE_DIR:=$(1)) + $(eval INCLUDE_DIRS+=-I$(1)) + $(eval CDFLIB_DIR:=$(1)) + $(eval CDFLIB_OBJ:=$(1)/cdflib.cpp.o) + $(eval PREBUILD_JOBS+=cdflib_obj) + + $(eval cdflib_obj: $(CDFLIB_OBJ)) + $(eval $(CDFLIB_OBJ): ; \ + cd $(CDFLIB_DIR) && $(CXX) $(CPP_FLAGS) $(OPTIMIZATION_FLAGS) $(DEFINE_FLAGS) $(INCLUDE_DIRS) -c cdflib.cpp -o cdflib.cpp.o) +endef + +# Add REFRESH - parallel queues monitor +define ADD_REFRESH_PARALLEL_QUEUES_MONITOR + $(info *** Adding refresh - parallel queues monitor ***) + $(eval REFRESH_PARALLEL_QUEUES_MONITOR_DIR:=$(1)/refresh/parallel_queues/lib/) + $(eval REFRESH_PARALLEL_QUEUES_MONITOR_OBJ:=$(1)/refresh/parallel_queues/lib/parallel-queues-monitor.cpp.o) + $(eval PREBUILD_JOBS+=refresh_parallel_queues_monitor_obj) + $(eval refresh_parallel_queues_monitor_obj: $(REFRESH_PARALLEL_QUEUES_MONITOR_OBJ)) + $(eval $(REFRESH_PARALLEL_QUEUES_MONITOR_OBJ): ; \ + cd $(REFRESH_PARALLEL_QUEUES_MONITOR_DIR) && $(CXX) $(CPP_FLAGS) $(OPTIMIZATION_FLAGS) $(DEFINE_FLAGS) $(INCLUDE_DIRS) -c parallel-queues-monitor.cpp -o parallel-queues-monitor.cpp.o) +endef + +# Add RADULS-inplace +define ADD_RADULS_INPLACE + $(info *** Adding raduls-inplace ***) + $(eval INCLUDE_DIRS+=-I$(1)/Raduls) + $(eval RADULS_INPLACE_DIR:=$(1)/Raduls) + $(eval RADULS_INPLACE_A_DIR:=$(1)/Raduls) + $(eval RADULS_INPLACE_A:=$(1)/Raduls/libraduls.a) + $(eval LIBRARY_FILES+=$(RADULS_INPLACE_A)) + $(eval LINKER_DIRS+=-L $(RADULS_INPLACE_A_DIR)) + $(eval PREBUILD_JOBS+=raduls-inplace) + + $(eval raduls-inplace: $(RADULS_INPLACE_A)) + $(eval $(RADULS_INPLACE_A) : ; \ + cd $(RADULS_INPLACE_DIR) && $(MAKE)) +endef + +# Add igraph +define ADD_IGRAPH + $(info *** Adding igraph ***) + $(eval INCLUDE_DIRS+=-I$(1)/include -I$(1)/build/include) + $(eval IGRAPH_DIR:=$(1)) + $(eval IGRAPH_A_DIR:=$(1)/build/src) + $(eval IGRAPH_A:=$(IGRAPH_A_DIR)/libigraph.a) + $(eval LIBRARY_FILES+=$(IGRAPH_A)) + $(eval LINKER_DIRS+=-L $(IGRAPH_A_DIR)) + $(eval IGRAPH_TARGET:=igraph) + $(call TEST_SOFT,cmake) + $(call TEST_SOFT,bison) + $(call TEST_SOFT,flex) + $(eval PREBUILD_JOBS+=igraph) + + $(eval igraph: $(IGRAPH_A)) + $(eval $(IGRAPH_A): ; \ + $(if $(filter Darwin,$(OS_TYPE)), \ + $(eval IEEE754_DOUBLE_ENDIANNESS_MATCHES_FIX:=-DIEEE754_DOUBLE_ENDIANNESS_MATCHES=TRUE), \ + $(eval IEEE754_DOUBLE_ENDIANNESS_MATCHES_FIX:=) \ + ) \ + mkdir -p $(IGRAPH_DIR)/build && cmake $(CMAKE_OSX_FIX) $(IEEE754_DOUBLE_ENDIANNESS_MATCHES_FIX) -DCMAKE_CXX_COMPILER=$(CXX) -DCMAKE_C_COMPILER=$(CC) -S $(IGRAPH_DIR) -B $(IGRAPH_DIR)/build && cmake --build $(IGRAPH_DIR)/build ) +endef + +# Add SBWT +define ADD_SBWT + $(info *** Adding SBWT ***) + $(eval INCLUDE_DIRS+=-I$(1)/include -I$(1)/sdsl-lite/include -I$(1)/SeqIO/include -I$(1)/build/external/sdsl-lite/build/external/libdivsufsort/include/) + $(eval SBWT_DIR:=$(1)) + $(eval SBWT_A_DIR:=$(1)/build) + $(eval SBWT_A:=$(SBWT_A_DIR)/libsbwt_static.a) + $(eval SBWT_SDSL_A:=$(SBWT_A_DIR)/external/sdsl-lite/build/lib/libsdsl.a) + $(eval SBWT_KMC_CORE_A:=$(SBWT_A_DIR)/external/KMC/build/libkmc_core.a) + $(eval SBWT_KMC_TOOLS_A:=$(SBWT_A_DIR)/external/KMC/build/libkmc_tools.a) + $(eval LIBRARY_FILES+=$(SBWT_A) $(SBWT_SDSL_A) $(SBWT_KMC_CORE_A) $(SBWT_KMC_TOOLS_A)) + $(eval LINKER_DIRS+=-L $(SBWT_A_DIR)) + $(eval PREBUILD_JOBS+=sbwt) + + $(eval sbwt: $(SBWT_A) $(SBWT_SDSL_A) $(SBWT_KMC_CORE_A) $(SBWT_KMC_TOOLS_A)) + $(eval $(SBWT_A): ; \ + mkdir -p $(SBWT_DIR)/build && cd $(SBWT_DIR)/build && cmake $(CMAKE_OSX_FIX) -DCMAKE_CXX_COMPILER=$(CXX) -DCMAKE_C_COMPILER=$(CC) .. -DMAX_KMER_LENGTH=32 && $(MAKE) -j) + $(eval $(SBWT_SDSL_A) : $(SBWT_A)) + $(eval $(SBWT_KMC_CORE_A) : $(SBWT_A)) + $(eval $(SBWT_KMC_TOOLS_A) : $(SBWT_A)) +endef + +# Add Pybind11 +define ADD_PYBIND11 + $(eval PYBIND11_DIR:=$(1)) + $(eval INCLUDE_DIRS+=-I$(PYBIND11_DIR)) + $(eval INCLUDE_DIRS+=-I $(shell python3 -c "import sysconfig;print(sysconfig.get_paths()['include'])")) + $(eval PY_EXTENSION_SUFFIX:=$(shell python3-config --extension-suffix)) +endef + +# Add REFRESH libs +define ADD_REFRESH_LIB + $(info *** Adding REFRESH libs ***) + $(eval REFRESH_DIR:=-I$(1)) +endef + +# Add StatsLib +define ADD_STATS_LIB + $(info *** Adding StatsLib ***) + $(eval INCLUDE_DIRS+=-I$(1)/include) +endef + +# Add Annoy +define ADD_ANNOY_LIB + $(info *** Adding Annoy lib ***) + $(eval INCLUDE_DIRS+=-I$(1)/include) +endef + +# Add hnswlib +define ADD_HNSWLIB + $(info *** Adding hnswlib ***) + $(eval INCLUDE_DIRS+=-I$(1)) +endef + +# Add umappp lib +define ADD_UMAPPP_LIB + $(info *** Adding UMAPPP lib ***) + $(eval INCLUDE_DIRS+=-I$(1)/include) +endef + +# Add CppIrlba lib +define ADD_CPPIRLBA_LIB + $(info *** Adding CppIrlba lib ***) + $(eval INCLUDE_DIRS+=-I$(1)/include) +endef + +# Add CppKmeans lib +define ADD_CPPKMEANS_LIB + $(info *** Adding CppIrlba lib ***) + $(eval INCLUDE_DIRS+=-I$(1)/include) +endef + +# Add aarand lib +define ADD_AARAND_LIB + $(info *** Adding aarand lib ***) + $(eval INCLUDE_DIRS+=-I$(1)/include) +endef + +# Add knncolle lib +define ADD_KNNCOLLE_LIB + $(info *** Adding knncolle lib ***) + $(eval INCLUDE_DIRS+=-I$(1)/include) +endef + +# Add Eigen lib +define ADD_EIGEN_LIB + $(info *** Adding Eigen lib ***) + $(eval INCLUDE_DIRS+=-I$(1)) +endef + +# Add NASM +# Idea is that if in Makefile ADD_NASM is called it will override the system one +define ADD_NASM + $(info *** Adding NASM ***) + $(eval export PATH := $(abspath $(1)):$(PATH)) + $(if $(filter x86_64,$(ARCH_TYPE)), \ + $(eval dummy_install_nasm:=$(shell \ + if [ ! -f $(1)/nasm ]; then \ + cd $(1) && ./autogen.sh && ./configure && $(MAKE) -j; \ + fi) \ + ) \ + $(eval NASM:=$(1)/nasm) \ + ) +endef + +### Macros configuring compiler/linker flags +# Add os-specific flags for static linking +define SET_STATIC + $(if $(filter true,$(1)), \ + $(if $(filter Darwin,$(OS_TYPE)), \ + $(eval STATIC_LFLAGS:=-static-libgcc -static-libstdc++ -pthread), \ + $(if $(filter x86_64,$(ARCH_TYPE)), \ + $(eval STATIC_LFLAGS:=-static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive), \ + $(eval STATIC_LFLAGS:=-static -Wl,--whole-archive -lpthread -Wl,--no-whole-archive) \ + ) + ) + ) +endef + +# Add C, C++ standards +define SET_C_CPP_STANDARDS + $(eval C_STD:=$(1)) + $(eval CPP_STD:=$(2)) +endef + +# Define allowed compiler version and type +define SET_COMPILER_VERSION_ALLOWED + $(eval COMPILER_VERSION_$(strip $(1))_$(strip $(2))_MIN:=$(strip $(3))) + $(eval COMPILER_VERSION_$(strip $(1))_$(strip $(2))_MAX:=$(strip $(4))) + $(eval COMPILER_ALLOWED+=COMPILER_VERSION_$(strip $(1))_$(strip $(2))) +endef + +# Set source, object and binary directories +define SET_SRC_OBJ_BIN + $(eval SRC_DIR:=$(1)) + $(eval OBJ_DIR:=$(2)) + $(eval OUT_BIN_DIR:=$(3)) +endef + +# *** Utility functions +define LESS_THAN + $(if $(filter 0,$(shell [ $(1) -lt $(2) ]; echo $$?)),1,0) +endef + +define GREATER_THAN + $(if $(filter 0,$(shell [ $(1) -gt $(2) ]; echo $$?)),1,0) +endef + +define IN_RANGE + $(shell if [ $(COMP) -ge $(MIN) ] && [ $(COMP) -le $(MAX) ]; then echo 1; else echo 0; fi) +endef + +define TEST_SOFT + $(if $(shell command -v $(1) >/dev/null 2>&1 && echo found),, \ + $(error The required software '$(1)' is not installed or not in PATH)) +endef + +# Check Git commit id and set GIT_COMMIT macro for compilation rule +define SET_GIT_COMMIT + $(eval GIT_COMMIT:=$(shell git describe --always --dirty)) + $(eval DEFINE_FLAGS:=-DGIT_COMMIT=$(GIT_COMMIT)) +endef + +# Prepare file variables +define LOAD_FILES +$(eval SRC_$(1)_DIR := $(SRC_DIR)/$(2)) +$(eval OBJ_$(1)_DIR := $(OBJ_DIR)/$(2)) +$(eval SRC_$(1) := $(wildcard $(SRC_$(1)_DIR)/*.cpp)) +$(eval OBJ_$(1) := $(patsubst $(SRC_$(1)_DIR)/%.cpp, $(OBJ_$(1)_DIR)/%.cpp.o, $(SRC_$(1)))) +endef + +# Dynamic creation of build rules +define DEFAULT_COMPILE_RULE = +$(OBJ_$(1)_DIR)/%.cpp.o: $(SRC_$(1)_DIR)/%.cpp | prebuild + @mkdir -p $(OBJ_$(1)_DIR) + $(CXX) $(CPP_FLAGS) $(OPTIMIZATION_FLAGS) $(DEFINE_FLAGS) $(INCLUDE_DIRS) -MMD -MF $$@.d -c $$< -o $$@ +endef + +# Dynamic creation of build rules for files in directory +define PREPARE_DEFAULT_COMPILE_RULE +# Source files +$(eval SRC_$(1)_DIR := $(SRC_DIR)/$(2)) +$(eval OBJ_$(1)_DIR := $(OBJ_DIR)/$(2)) +$(eval SRC_$(1) := $(wildcard $(SRC_$(1)_DIR)/*.cpp)) +$(eval OBJ_$(1) := $(patsubst $(SRC_$(1)_DIR)/%.cpp, $(OBJ_$(1)_DIR)/%.cpp.o, $(SRC_$(1)))) +# Compilation rule +$(OBJ_$(1)_DIR)/%.cpp.o: $(SRC_$(1)_DIR)/%.cpp | prebuild + mkdir -p $(OBJ_$(1)_DIR) + $(CXX) $(3) $(CPP_FLAGS) $(OPTIMIZATION_FLAGS) $(ARCH_FLAGS) $(DEFINE_FLAGS) $(INCLUDE_DIRS) -MMD -MF $$@.d -c $$< -o $$@ +# Dependency files +-include $(OBJ_$(1):.o=.o.d) +endef + +# Check compiler version +define CHECK_COMPILER_VERSION + $(info *** Checking compiler version ***) + $(eval COMPILER_DESC:=$(shell command -v $(CXX) >/dev/null 2>&1 && basename $(CXX) | sed 's/-.*//' || echo "")) + + $(if $(COMPILER_DESC),,\ + $(error Compiler does not exist) \ + ) + + $(eval COMPILER_VERSION_FULL:=$(shell $(CXX) --version | sed -n '1s/^[^0-9]*\([0-9\.]*\).*$$/\1/p')) + $(eval COMPILER_VERSION_MAJOR:=$(firstword $(subst ., ,$(COMPILER_VERSION_FULL)))) + + $(eval COMPILER_DESC:=$(subst g++,GCC,$(COMPILER_DESC))) + $(eval COMPILER_DESC:=$(subst clang,CLANG,$(COMPILER_DESC))) + + $(info Compiler: $(COMPILER_DESC)) + $(info Version: $(COMPILER_VERSION_MAJOR)) + + $(if $(or $(COMPILER_VERSION_$(COMPILER_DESC)_$(OS_ARCH_TYPE)_MIN),$(COMPILER_VERSION_$(COMPILER_DESC)_$(OS_ARCH_TYPE)_MAX)),\ + ,\ + $(error Compiler not supported) \ + ) + + $(if $(COMPILER_VERSION_$(COMPILER_DESC)_$(OS_ARCH_TYPE)_MIN), \ + $(if $(filter 1,$(call LESS_THAN,$(COMPILER_VERSION_MAJOR),$(COMPILER_VERSION_$(COMPILER_DESC)_$(OS_ARCH_TYPE)_MIN))), \ + $(error Too low compiler version), \ + $(if $(COMPILER_VERSION_$(COMPILER_DESC)_$(OS_ARCH_TYPE)_MAX), \ + $(if $(filter 1,$(call GREATER_THAN,$(COMPILER_VERSION_MAJOR),$(COMPILER_VERSION_$(COMPILER_DESC)_$(OS_ARCH_TYPE)_MAX))), \ + $(error Too high compiler version) \ + ), \ + ) + ), \ + ) +endef + +# Add type-specifix flags +define SET_FLAGS + $(if $(filter Linux_x86_64,$(OS_ARCH_TYPE)), \ + $(eval PLATFORM_SPECIFIC_C_FLAGS:=) \ + $(eval PLATFORM_SPECIFIC_CPP_FLAGS:=) \ + $(eval PLATFORM_SPECIFIC_LINKER_FLAGS:=-fabi-version=6), \ + $(if $(filter Linux_aarch64,$(OS_ARCH_TYPE)), \ + $(eval PLATFORM_SPECIFIC_C_FLAGS:=) \ + $(eval PLATFORM_SPECIFIC_CPP_FLAGS:=-ffp-contract=off) \ + $(eval PLATFORM_SPECIFIC_LINKER_FLAGS:=-fabi-version=6), \ + $(if $(filter Darwin_arm64,$(OS_ARCH_TYPE)), \ + $(eval PLATFORM_SPECIFIC_C_FLAGS:=) \ + $(eval PLATFORM_SPECIFIC_CPP_FLAGS:=) \ + $(eval PLATFORM_SPECIFIC_LINKER_FLAGS:=), \ + $(if $(filter Darwin_x86_64,$(OS_ARCH_TYPE)), \ + $(eval PLATFORM_SPECIFIC_C_FLAGS:=) \ + $(eval PLATFORM_SPECIFIC_CPP_FLAGS:=) \ + $(eval PLATFORM_SPECIFIC_LINKER_FLAGS:=) \ + ) \ + ) \ + ) \ + ) + + $(eval C_FLAGS+=-std=$(C_STD) -Wall -fPIC -pthread -fpermissive $(PLATFORM_SPECIFIC_C_FLAGS)) + $(eval CPP_FLAGS+=-std=$(CPP_STD) -Wall -fPIC -pthread -fpermissive $(PLATFORM_SPECIFIC_CPP_FLAGS)) + $(eval LINKER_FLAGS+=-lm -lpthread $(PLATFORM_SPECIFIC_LINKER_FLAGS) $(STATIC_LFLAGS)) + $(eval PY_FLAGS:=-Wl,-undefined,dynamic_lookup -shared) + + + $(if $(filter release,$(1)), \ + $(eval OPTIMIZATION_FLAGS+=-O3) \ + $(eval C_FLAGS+=) \ + $(eval CPP_FLAGS+= ), \ + $(if $(filter debug,$(1)), \ + $(eval OPTIMIZATION_FLAGS+=-O0 -g) \ + $(eval C_FLAGS+=) \ + $(eval CPP_FLAGS+= ), \ + $(if $(filter ASan,$(1)), \ + $(eval OPTIMIZATION_FLAGS+=-O3 -g) \ + $(eval C_FLAGS+=-fsanitize=address) \ + $(eval CPP_FLAGS+=-fsanitize=address) \ + $(eval LINKER_FLAGS+=-fsanitize=address), \ + $(if $(filter TSan,$(1)), \ + $(eval OPTIMIZATION_FLAGS+=-O3 -g) \ + $(eval C_FLAGS+=-fsanitize=thread) \ + $(eval CPP_FLAGS+=-fsanitize=thread) \ + $(eval LINKER_FLAGS+=-fsanitize=thread -static-libgcc -static-libstdc++), \ + $(if $(filter UBSan,$(1)), \ + $(eval OPTIMIZATION_FLAGS+=-O3 -g) \ + $(eval C_FLAGS+=-fsanitize=undefined) \ + $(eval CPP_FLAGS+=-fsanitize=undefined) \ + $(eval LINKER_FLAGS+=-fsanitize=undefined), \ + $(if $(filter LSan,$(1)), \ + $(eval OPTIMIZATION_FLAGS+=-O3 -g) \ + $(eval C_FLAGS+=-fsanitize=leak) \ + $(eval CPP_FLAGS+=-fsanitize=leak) \ + $(eval LINKER_FLAGS+=-fsanitize=leak), \ + $(if $(filter MSan,$(1)), \ + $(eval OPTIMIZATION_FLAGS+=-O3 -g) \ + $(eval C_FLAGS+=-fsanitize=memory) \ + $(eval CPP_FLAGS+=-fsanitize=memory) \ + $(eval LINKER_FLAGS+=-fsanitize=memory), \ + ) \ + ) \ + ) \ + ) \ + ) \ + ) \ + ) + + $(eval CPP_FLAGS_SSE2:=$(CPPFLAGS) -msse2) + $(eval CPP_FLAGS_SSE4:=$(CPPFLAGS) -msse4) + $(eval CPP_FLAGS_AVX:=$(CPPFLAGS) -mavx) + $(eval CPP_FLAGS_AVX2:=$(CPPFLAGS) -mavx2) + $(eval CPP_FLAGS_AVX512:=$(CPPFLAGS) -mavx512) + $(eval CPP_FLAGS_NEON:=$(CPPFLAGS)) + + $(eval INCLUDE_DIRS+=$(REFRESH_DIR)) + $(info Prebuild jobs: $(PREBUILD_JOBS)) +prebuild: + $(PREBUILD_JOBS) +endef + +### Macros checking system and software +# Check for NASM +define CHECK_NASM + $(eval NASM_VERSION:=$(shell $(NASM) --version 2>/dev/null)) +endef + +# Choose lib for gzip decompression +define CHOOSE_GZIP_DECOMPRESSION + $(if $(filter x86_64,$(ARCH_TYPE)), \ + $(if $(and $(NASM_VERSION),$(ISAL_DIR)), \ + $(eval GZ_TARGET:=isa-l) \ + $(eval PREBUILD_JOBS+=isa-l) \ + $(eval INCLUDE_DIRS+=-I$(ISAL_DIR)/include) ,\ + $(eval GZ_TARGET:=zlib-ng) \ + $(eval PREBUILD_JOBS+=zlib-ng) \ + $(eval INCLUDE_DIRS+=-I$(ZLIB_DIR)/build-g++ -I$(ZLIB_DIR)/build-g++/zlib-ng) \ + ), \ + $(eval GZ_TARGET:=zlib-ng) \ + $(eval PREBUILD_JOBS+=zlib-ng) \ + $(eval INCLUDE_DIRS+=-I$(ZLIB_DIR)/build-g++ -I$(ZLIB_DIR)/build-g++/zlib-ng) \ + ) + + $(if $(filter isa-l,$(GZ_TARGET)), \ + $(info ISAL will be used for gzip decompression) \ + $(eval GZ_LIB:=isa-l.a) \ + $(eval LIBRARY_FILES+=$(ISAL_A)) \ + $(eval LINKER_DIRS+=-L $(ISAL_A_DIR)) + $(eval C_FLAGS+=-DREFRESH_USE_IGZIP) \ + $(eval CPP_FLAGS+=-DREFRESH_USE_IGZIP), \ + $(info zlib-ng will be used for gzip decompression) \ + $(eval GZ_LIB:=libz.a) \ + $(eval LIBRARY_FILES+=$(ZLIB_A)) \ + $(eval LINKER_DIRS+=-L $(ZLIB_A_DIR)) + $(eval C_FLAGS+=-DREFRESH_USE_ZLIB) \ + $(eval CPP_FLAGS+=-DREFRESH_USE_ZLIB) \ + ) +endef + +# Check for OS and architecture +define CHECK_OS_ARCH + $(if $(MSVC), \ + $(eval OS_TYPE:=windows) \ + $(eval ARCH_TYPE:=x86_64), \ + $(eval OS_TYPE:=$(shell uname -s 2>/dev/null || echo not)) \ + $(eval ARCH_TYPE:=$(shell uname -m 2>/dev/null || echo not)) \ + ) + + $(eval OS_ARCH_TYPE:=$(OS_TYPE)_$(ARCH_TYPE)) + + $(if $(filter arm8,$(1)), \ + $(eval ARCH_FLAGS:=-march=armv8-a -DARCH_ARM) \ + $(info *** ARMv8 with NEON extensions ***), \ + $(if $(filter m1,$(1)), \ + $(eval ARCH_FLAGS:=-march=armv8.4-a -DARCH_ARM) \ + $(info *** Apple M1 (or newer) with NEON extensions ***), \ + $(if $(filter sse2,$(1)), \ + $(eval ARCH_FLAGS:=-msse2 -m64 -DARCH_X64) \ + $(info *** x86-64 with SSE2 extensions ***), \ + $(if $(filter avx,$(1)), \ + $(eval ARCH_FLAGS:=-mavx -m64 -DARCH_X64) \ + $(info *** x86-64 with AVX extensions ***), \ + $(if $(filter avx2,$(1)), \ + $(eval ARCH_FLAGS:=-mavx2 -m64 -DARCH_X64) \ + $(info *** x86-64 with AVX2 extensions ***), \ + $(if $(filter avx512,$(1)), \ + $(eval ARCH_FLAGS:=-mavx512 -m64 -DARCH_X64) \ + $(info *** x86-64 with AVX512 extensions ***), \ + $(if $(filter x86_64,$(ARCH_TYPE)), \ + $(eval ARCH_FLAGS:=-march=native -DARCH_X64) \ + $(info *** Unspecified platform - using native compilation for x86_64 ***), \ + $(eval ARCH_FLAGS:=-march=native -DARCH_ARM) \ + $(info *** Unspecified platform - using native compilation for ARM ***)))))))) + + $(if $(filter Darwin,$(OS_TYPE)), \ + $(eval SDK_PATH:=$(shell $(CXX) -v 2>&1 | grep -- '--with-sysroot' | sed -E 's/.*--with-sysroot=([^ ]+).*/\1/')) \ + $(eval CMAKE_OSX_FIX:=-DCMAKE_OSX_SYSROOT=$(SDK_PATH)) \ + ) + + $(if $(filter Darwin,$(OS_TYPE)), \ + $(eval AR_OPT:=-rcs) \ + $(eval PY_AGC_API_CFLAGS:=-Wl,-undefined,dynamic_lookup -fPIC -Wall -shared -std=c++14 -O3), \ + $(eval AR_OPT:=rcs -o) \ + $(eval PY_AGC_API_CFLAGS:=-fPIC -Wall -shared -std=c++14 -O3) \ + ) +endef + +# Load submodules if necessary +# $(eval dummy:=$(shell git submodule update --init --recursive)) + +define INIT_SUBMODULES_FAST + $(info *** Initialization of submodules (fast) ***) + $(if $(shell git submodule status | grep '^-'), \ + $(info Initializing and updating submodules...) \ + $(eval dummy:=$(shell git submodule update --init --recursive --jobs=8)), \ + $(info Submodules are already up-to-date or none exist.) + ) +endef + +define INIT_SUBMODULES + $(info *** Initialization of submodules ***) + $(eval dummy:=$(shell git submodule update --init --recursive --jobs=8)) +endef + +### Clean library targets +clean-zlib-ng: + -cd $(ZLIB_DIR) && $(MAKE) -f Makefile.in clean && rm -r build-g++ + +clean-isa-l: + -cd $(ISAL_DIR) && $(MAKE) -f Makefile.unx clean + +clean-libdeflate: + -cd $(LIBDEFLATE_DIR) && rm -r build + +clean-libzstd: + -cd $(LIBZSTD_DIR) && $(MAKE) clean + +clean-raduls-inplace: + -cd $(RADULS_INPLACE_DIR) && $(MAKE) clean + +clean-igraph: + -rm -r $(IGRAPH_DIR)/build + +clean-mimalloc_obj: + -rm $(MIMALLOC_OBJ) + +clean-cdflib_obj: + -rm $(CDFLIB_OBJ) + +clean-refresh_parallel_queues_monitor_obj: + -rm $(CDFLIB_OBJ) + +clean-sbwt: + -rm $(SBWT_A) + -rm $(SBWT_SDSL_A) + -rm $(SBWT_KMC_CORE_A) + -rm $(SBWT_KMC_TOOLS_A) + -rm -r $(SBWT_A_DIR) + +### Testing +define show_var + $(info $(1): $($(1))) +endef + +define show_var_opt + $(if $(1), \ + $(info $(1): $($(1))) \ + ) +endef + +_testing: + $(info *** General ***) + $(call show_var,OS_TYPE) + $(call show_var,ARCH_TYPE) + $(call show_var,OS_ARCH_TYPE) + $(call show_var,ARCH_FLAGS) + $(call show_var,NASM_VERSION) + + $(info *** Compilers ***) + $(call show_var,COMPILER_DESC) + $(call show_var,COMPILER_VERSION_FULL) + $(call show_var,COMPILER_VERSION_MAJOR) + $(call show_var,COMPILER_ALLOWED) + $(foreach desc,\ + $(wordlist 1,$(words $(COMPILER_ALLOWED)),$(COMPILER_ALLOWED)), \ + $(call show_var,$(desc)_MIN) \ + $(call show_var,$(desc)_MAX) \ + ) + + $(info *** Main directories ***) + $(call show_var,INCLUDE_DIRS) + $(call show_var,LIBRARY_DIRS) + + $(info *** Compiler and linker flags ***) + $(call show_var,C_STD) + $(call show_var,CPP_STD) + $(call show_var,C_FLAGS) + $(call show_var,CPP_FLAGS) + $(call show_var,OPTIMIZATION_FLAGS) + $(call show_var,DEFINE_FLAGS) + $(call show_var,LINKER_FLAGS) + $(call show_var,STATIC_LFLAGS) + $(call show_var,CPP_FLAGS_SSE2) + $(call show_var,CPP_FLAGS_SSE4) + $(call show_var,CPP_FLAGS_AVX) + $(call show_var,CPP_FLAGS_AVX2) + $(call show_var,CPP_FLAGS_AVX512) + $(call show_var,CPP_FLAGS_NEON) + + $(info *** Files ***) + $(call show_var,SRC_DIR) + $(call show_var,OBJ_DIR) + $(call show_var,OUT_BIN_DIR) + $(call show_var,FILES_DEFINED) + $(foreach item,\ + $(wordlist 1,$(words $(FILES_DEFINED)),$(FILES_DEFINED)), \ + $(call show_var,SRC_$(item)_DIR) \ + $(call show_var,OBJ_$(item)_DIR) \ + $(call show_var,SRC_$(item)) \ + $(call show_var,OBJ_$(item)) \ + ) + + $(info *** Libraries ***) + $(info * gzip decompression *) + $(call show_var,GZ_TARGET) + + $(info * zlib-ng *) + $(call show_var_opt,ZLIB_DIR) + $(call show_var_opt,ZLIB_A_DIR) + $(call show_var_opt,ZLIB_A) + + $(info * isa-l *) + $(call show_var_opt,ISAL_DIR) + $(call show_var_opt,ISAL_A_DIR) + $(call show_var_opt,ISAL_A) + + $(info * libdeflate *) + $(call show_var_opt,LIBDEFLATE_DIR) + $(call show_var_opt,LIBDEFLATE_A_DIR) + $(call show_var_opt,LIBDEFLATE_A) + + $(info * libzstd *) + $(call show_var_opt,LIBZSTD_DIR) + $(call show_var_opt,LIBZSTD_A_DIR) + $(call show_var_opt,LIBZSTD_A) + + $(info * mimalloc *) + $(call show_var_opt,MIMALLOC_INCLUDE_DIR) + $(call show_var_opt,MIMALLOC_DIR) + $(call show_var_opt,MIMALLOC_OBJ) + + $(info * raduls *) + $(call show_var_opt,RADULS_INPLACE_DIR) + $(call show_var_opt,RADULS_INPLACE_A_DIR) + $(call show_var_opt,RADULS_INPLACE_A) + + $(info * igraph *) + $(call show_var_opt,IGRAPH_DIR) + $(call show_var_opt,IGRAPH_A_DIR) + $(call show_var_opt,IGRAPH_A) + + $(info * SBWT *) + $(call show_var_opt,SBWT_DIR) + $(call show_var_opt,SBWT_A_DIR) + $(call show_var_opt,SBWT_A) + $(call show_var_opt,SBWT_SDSL_A) + $(call show_var_opt,SBWT_KMC_CORE_A) + $(call show_var_opt,SBWT_KMC_TOOLS_A) + diff --git a/src/ani-entropy.vcxproj b/src/ani-entropy.vcxproj index 5e06052..da7f1c7 100644 --- a/src/ani-entropy.vcxproj +++ b/src/ani-entropy.vcxproj @@ -176,6 +176,7 @@ + diff --git a/src/ani-entropy.vcxproj.filters b/src/ani-entropy.vcxproj.filters index 2b48c01..a87e9fa 100644 --- a/src/ani-entropy.vcxproj.filters +++ b/src/ani-entropy.vcxproj.filters @@ -51,6 +51,9 @@ Library Files + + Library Files + diff --git a/src/defs.h b/src/defs.h index 8deb62c..b95040d 100644 --- a/src/defs.h +++ b/src/defs.h @@ -15,9 +15,9 @@ #include #include "params.h" -const std::string LZ_ANI_VER = "lz-ani 1.2.2"; -const std::string LZ_ANI_VERSION = "1.2.2"; -const std::string LZ_ANI_DATE = "2024-10-22"; +const std::string LZ_ANI_VER = "lz-ani 1.2.3"; +const std::string LZ_ANI_VERSION = "1.2.3"; +const std::string LZ_ANI_DATE = "2024-11-02"; const std::string LZ_ANI_AUTHORS = "Sebastian Deorowicz, Adam Gudys"; const std::string LZ_ANI_INFO = LZ_ANI_VER + " (" + LZ_ANI_DATE + ") by " + LZ_ANI_AUTHORS; diff --git a/src/filter.cpp b/src/filter.cpp index 5b186ca..d8f16d3 100644 --- a/src/filter.cpp +++ b/src/filter.cpp @@ -14,6 +14,7 @@ #include "utils.h" #include "../libs/refresh/parallel_queues/lib/parallel-queues.h" #include "../libs/refresh/conversions/lib/numeric_conversions.h" +#include "../libs/refresh/logs/lib/progress.h" // **************************************************************************** bool CFilter::load_filter(const string& fn, double thr, uint32_t no_threads, uint32_t verbosity_level) @@ -29,8 +30,6 @@ bool CFilter::load_filter(const string& fn, double thr, uint32_t no_threads, uin refresh::stream_decompression sfil(&fil, 16 << 20); string line; - vector parts; - vector elem; sfil.getline(line); // genome names sequence_names = split(line, ','); @@ -47,13 +46,18 @@ bool CFilter::load_filter(const string& fn, double thr, uint32_t no_threads, uin line.clear(); line.shrink_to_fit(); - uint64_t no_items = 0; + atomic no_items = 0; if(verbosity_level >= 1) cerr << "Loading filter data" << endl; - if (no_threads < 4) + refresh::progress_state progress(sequence_names.size(), "", "%", -1); + + if (no_threads < 2) { + vector parts; + vector elem; + for (int i = 0; !sfil.eof(); ++i) { sfil.getline(line); @@ -81,27 +85,36 @@ bool CFilter::load_filter(const string& fn, double thr, uint32_t no_threads, uin filter[i].shrink_to_fit(); no_items += filter[i].size(); + + if (progress.increment(1) && verbosity_level >= 2) + cerr << progress.str() << "\r"; } } else { - refresh::parallel_queue pq_lines(128, 1); - refresh::parallel_queue>> pq_parts(128, 1); - refresh::parallel_queue> pq_ids(128, 1); + refresh::parallel_queue> pq_lines(128 + 2 * no_threads, 1); +// refresh::parallel_queue>> pq_parts(128, 1); +// refresh::parallel_queue> pq_ids(128, 1); thread thr_reader([&] { + string line; + uint32_t id = 0; + while (!sfil.eof()) { - string line; sfil.getline(line); if (line.length() > 2) - pq_lines.push(move(line)); + { + pq_lines.push(move(make_pair(id++, line))); + line.clear(); + } } pq_lines.mark_completed(); }); +#if 0 thread thr_splitter([&] { string line; @@ -185,14 +198,62 @@ bool CFilter::load_filter(const string& fn, double thr, uint32_t no_threads, uin } cerr << row_id << "\n"; - thr_reader.join(); thr_splitter.join(); thr_converter.join(); +#endif + vector thrs_worker; + + for (uint32_t i = 0; i < no_threads - 1; ++i) + thrs_worker.emplace_back([&] + { + pair id_line; + vector parts; + vector elem; + uint64_t local_no_items = 0; - vector first_pass_sizes(filter.size(), 0); + while (pq_lines.pop(id_line)) + { + parts = split(id_line.second, ','); + + if (parts.size() > 1) + { + for (size_t j = 1; j < parts.size(); ++j) + { + const auto p = parts[j]; + elem = split(p, ':'); + if (elem.size() == 2) + { + double val = stod(elem[1]); + + if (val >= thr) + filter[id_line.first].emplace_back(stoi(elem[0]) - 1); + } + } + + filter[id_line.first].shrink_to_fit(); + local_no_items += filter[id_line.first].size(); + } + + if (progress.increment_ts(1) && verbosity_level >= 2) + cerr << progress.str_ts() + "\r"s; + } + + no_items += local_no_items; + }); + + thr_reader.join(); + for (auto& t : thrs_worker) + t.join(); + + vector first_pass_sizes(sequence_names.size(), 0); if(verbosity_level >= 2) - cerr << "End of loading data" << endl; + cerr << endl << "End of loading data" << endl; + + vector no_items_to_extend(filter.size(), 0); + for (size_t i = 0; i < filter.size(); ++i) + for (auto x : filter[i]) + ++no_items_to_extend[x]; for (size_t i = 0; i < filter.size(); ++i) { @@ -225,12 +286,14 @@ bool CFilter::load_filter(const string& fn, double thr, uint32_t no_threads, uin }); for (auto& t : thr_workers) - t.join(); + t.join(); } if (verbosity_level >= 1) cerr << "Filter size: " << no_items << endl; + filter_size = no_items; + return true; } diff --git a/src/filter.h b/src/filter.h index b83450c..50cd637 100644 --- a/src/filter.h +++ b/src/filter.h @@ -23,6 +23,7 @@ class CFilter { vector> filter; vector sequence_names; + size_t filter_size; long int local_strtol(const char* str, char** endptr) { long int val = 0; @@ -79,6 +80,11 @@ class CFilter filter[i].clear(); filter[i].shrink_to_fit(); } + + size_t size() + { + return filter_size; + } }; // EOF diff --git a/src/lz_matcher.cpp b/src/lz_matcher.cpp index d4974db..31608cf 100644 --- a/src/lz_matcher.cpp +++ b/src/lz_matcher.cpp @@ -14,6 +14,7 @@ #include "parser.h" #include "../libs/refresh/parallel_queues/lib/parallel-queues.h" #include "../libs/refresh/conversions/lib/numeric_conversions.h" +#include "../libs/refresh/logs/lib/progress.h" // **************************************************************************** bool CLZMatcher::load_sequences() @@ -49,7 +50,22 @@ bool CLZMatcher::compare_sequences() if (seq_sn.size() != flt_sn.size() || seq_sn != flt_sn) { - cerr << "Input sequences and filter sequences are different!" << endl; + cerr << "seq_sn.size(): " << seq_sn.size() << endl; + cerr << "flt_sn.size(): " << flt_sn.size() << endl; + + cerr << "*** seq_sn" << endl; + for (auto x : seq_sn) + cerr << x << endl; + + cerr << "*** flt_sn" << endl; + for (auto x : flt_sn) + cerr << x << endl; + + + if(seq_sn.size() != flt_sn.size()) + cerr << "Input sequences and filter sequences sets are of different size!" << endl; + else + cerr << "Input sequences and filter sequences are different!" << endl; return false; } @@ -168,6 +184,9 @@ void CLZMatcher::do_matching() atomic global_task_no = 0; atomic global_no_pairs = 0; + refresh::progress_state ps_seq(seq_reservoir.size(), "Sequences: ", "%", -1); + refresh::progress_state ps_pair(filter.is_empty() ? seq_reservoir.size() * (seq_reservoir.size() - 1) : filter.size(), "Pairs: ", "%", -1); + for (uint32_t i = 0; i < params.no_threads; ++i) { thr_workers.emplace_back([&] { @@ -189,15 +208,11 @@ void CLZMatcher::do_matching() auto sr_iter = seq_reservoir.get_sequence(local_task_no); parser.prepare_reference(seq_view(sr_iter->data, sr_iter->len, params.internal_packing), sr_iter->no_parts); - uint64_t to_add = filter.is_empty() ? local_task_no : (uint64_t)filter.get_row(local_task_no).size(); - auto to_print = global_no_pairs.fetch_add(to_add); - - if (params.verbosity_level >= 2) - cerr << to_string(local_task_no) + " : " + to_string(to_print) + " \r"; +// uint64_t to_add = filter.is_empty() ? local_task_no : (uint64_t)filter.get_row(local_task_no).size(); +// auto to_print = global_no_pairs.fetch_add(to_add); if (filter.is_empty()) { -// for (uint64_t id = 0; id < local_task_no; ++id) for (uint64_t id = 0; id < seq_reservoir.size(); ++id) { if (id == local_task_no) @@ -232,14 +247,24 @@ void CLZMatcher::do_matching() res_row.emplace_back(id, results); } - - filter.clear_row(local_task_no); } res_row.shrink_to_fit(); sort(res_row.begin(), res_row.end()); results[local_task_no] = move(res_row); + + if (params.verbosity_level >= 2) + { + bool r1 = ps_seq.increment_ts(1); + bool r2 = ps_pair.increment_ts(filter.is_empty() ? (uint64_t)(seq_reservoir.size() - 1) : (uint64_t)filter.get_row(local_task_no).size()); + // cerr << to_string(local_task_no) + " : " + to_string(to_print) + " \r"; + if (r1 || r2) + cerr << ps_seq.str_ts() + " " + ps_pair.str_ts() + " \r" << std::flush; + } + + if (!filter.is_empty()) + filter.clear_row(local_task_no); } }); } @@ -532,8 +557,17 @@ bool CLZMatcher::store_results() string to_print; + refresh::progress_state ps(results.size(), "", "%", -1); + while (par_queue.pop(to_print)) + { ofs.write(to_print.data(), to_print.size()); + if (params.verbosity_level >= 2 && ps.increment(1)) + cerr << ps.str() << "\r" << std::flush; + } + + if (params.verbosity_level >= 2) + cerr << endl; for (auto& t : thr_workers) t.join(); diff --git a/src/seq_reservoir.cpp b/src/seq_reservoir.cpp index 0259d0b..274eaf1 100644 --- a/src/seq_reservoir.cpp +++ b/src/seq_reservoir.cpp @@ -9,6 +9,7 @@ // ******************************************************************************************* #include "seq_reservoir.h" +#include "../libs/refresh/logs/lib/progress.h" #include #include @@ -79,6 +80,9 @@ void CSeqReservoir::append(const string& name, const string& seq) copy(name.begin(), p, ptr_name); ptr_name[p - name.begin()] = 0; + if (items.size() == items.capacity()) + items.reserve(items.size() * 2); + items.emplace_back(ptr_name, ptr_seq, seq.length(), 1); } @@ -151,6 +155,8 @@ bool CSeqReservoir::load_fasta(const vector& fasta_files, uint32_t sep_l // **************************************************************************** bool CSeqReservoir::load_multifasta(const vector& fasta_files, uint32_t verbosity_level) { + refresh::progress_state progress; + for (const auto& fn : fasta_files) { refresh::stream_in_file sif(fn, 16 << 20, 16 << 20); @@ -177,27 +183,28 @@ bool CSeqReservoir::load_multifasta(const vector& fasta_files, uint32_t if (line.front() == '>') { if (!name.empty()) + { append(name, seq); + + if (verbosity_level >= 2 && items.size() % 1000 == 0 && progress.increment(1000)) + cerr << progress.str() + "\r"s << std::flush; + } name.assign(line.begin() + 1, line.end()); seq.clear(); } else seq.append(line); - - if (verbosity_level >= 2 && items.size() % 1000 == 0) - { - cerr << items.size() << "\r"; - fflush(stdout); - } } if (!name.empty()) append(name, seq); } + items.shrink_to_fit(); + if (verbosity_level >= 2) { - cerr << items.size() << "\r"; + cerr << items.size() << endl; fflush(stdout); }