diff --git a/.github/actions/setup-dependencies/action.yml b/.github/actions/setup-dependencies/action.yml index 72caef2eec..b246256862 100644 --- a/.github/actions/setup-dependencies/action.yml +++ b/.github/actions/setup-dependencies/action.yml @@ -3,13 +3,18 @@ inputs: description: "Python version to setup" required: false default: "3.9" + install-test-deps: + description: "Wether to install 3rd Party dependencies (for tests)" + required: false + default: true name: "Setup dependencies" description: "Install all required dependencies for worflows to run." runs: using: "composite" steps: - - name: Install 3rd party dependencies + - if: inputs.install-test-deps == 'true' + name: Install 3rd party dependencies run: sudo ./install-deps.sh shell: bash @@ -21,8 +26,12 @@ runs: with: enable-cache: true cache-dependency-glob: "uv.lock" - python-version: ${{ matrix.python-version }} + python-version: ${{ inputs.python-version }} - name: Install the project run: uv sync shell: bash + + - name: Setup pip + run: uv pip install pip + shell: bash diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 0000000000..90b6d81a95 --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,427 @@ +name: CI + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + merge_group: + workflow_dispatch: + inputs: + test_release: + description: If true, publish to test.pypi.org + required: true + default: true + type: boolean + +permissions: + contents: read + +env: + SCCACHE_GHA_ENABLED: "true" + RUSTC_WRAPPER: "sccache" + DOCKER_IMAGE: ghcr.io/onekey-sec/unblob + +jobs: + check_pre_commit: + name: Check - pre-commit + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + install-test-deps: "false" + + - name: Setup Nix + uses: cachix/install-nix-action@v30 + with: + install_url: https://releases.nixos.org/nix/nix-2.18.8/install + + - name: Check pre-commit hook + uses: pre-commit/action@v3.0.1 + + check_pyright: + name: Check - pyright + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + + - name: Check - pyright + run: uv run pyright . + + run_python_tests: + name: Run tests (Python) + needs: [check_pre_commit, check_pyright] + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + python-version: ${{ matrix.python-version }} + + - name: Setup git lfs + uses: ./.github/actions/setup-git-lfs + + - name: Run pytest + run: uv run pytest -vvv + + run_rust_tests: + name: Run tests (Rust) + needs: [check_pre_commit] + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + + - name: Setup sccache + uses: mozilla-actions/sccache-action@v0.0.7 + + - name: Run cargo test + run: cargo test + + build_linux_wheels: + name: Build wheels (linux) + if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [check_pre_commit] + runs-on: ubuntu-24.04 + strategy: + fail-fast: false + matrix: + platform: + - manylinux: manylinux2014 + target: x86_64 + - manylinux: manylinux2014 + target: aarch64 + - manylinux: musllinux_1_1 + target: x86_64 + # lief is not available for this platform (and no sdist is provided) + # - manylinux: musllinux_1_1 + # target: aarch64 + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist + sccache: "true" + manylinux: ${{ matrix.platform.manylinux }} + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.platform.manylinux }}-${{ matrix.platform.target }} + path: dist + - name: Check wheel (x86-manylinux) + if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') + shell: bash + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + pip install dist/*.whl + unblob --help + - name: Check wheel (cross-manylinux) + if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'manylinux') + uses: uraimo/run-on-arch-action@v2 + with: + arch: ${{ matrix.platform.target }} + distro: ubuntu22.04 + githubToken: ${{ github.token }} + install: | + apt-get update + apt-get install -y --no-install-recommends python3-venv libmagic1 + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + pip install dist/*.whl + unblob --version + - name: Check wheel (x86-musllinux) + if: matrix.platform.target == 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') + uses: addnab/docker-run-action@v3 + with: + image: alpine:latest + options: -v ${{ github.workspace }}:/io -w /io + run: | + set -e + apk add py3-pip libmagic gcc lz4 musl-dev python3-dev + python3 -m venv .venv + source .venv/bin/activate + pip install dist/*.whl + unblob --version + - name: Check wheel (cross-musllinux) + if: matrix.platform.target != 'x86_64' && startsWith(matrix.platform.manylinux, 'musllinux') + uses: uraimo/run-on-arch-action@v2 + with: + arch: ${{ matrix.platform.target }} + distro: alpine_latest + githubToken: ${{ github.token }} + install: | + apk add py3-pip libmagic gcc lz4 musl-dev python3-dev + run: | + set -e + python3 -m venv .venv + source .venv/bin/activate + pip install dist/*.whl + unblob --version + + build_macos_wheels: + name: Build wheels (macos) + if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [check_pre_commit] + runs-on: ${{ matrix.platform.runner }} + strategy: + fail-fast: false + matrix: + platform: + - runner: macos-13 + target: x86_64 + - runner: macos-14 + target: aarch64 + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + install-test-deps: "false" + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist + sccache: "true" + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-macos-${{ matrix.platform.target }} + path: dist + - name: Check wheel + run: | + set -e + brew install libmagic + python3 -m venv .venv + source .venv/bin/activate + pip install --find-links dist unblob + unblob --version + + build_sdist: + name: Build sdist + if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [run_python_tests, run_rust_tests] + runs-on: ubuntu-latest + steps: + - name: Checkout source code + uses: actions/checkout@v4 + - name: Setup 3rd party dependencies + uses: ./.github/actions/setup-dependencies + with: + install-test-deps: "false" + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist + + build-image: + name: Build Docker image + if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') + needs: [build_linux_wheels] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + platform: + - docker: linux/amd64 + wheel: wheels-manylinux2014-x86_64 + - docker: linux/arm64 + wheel: wheels-manylinux2014-aarch64 + steps: + - name: Download wheel + uses: actions/download-artifact@v4 + with: + name: ${{ matrix.platform.wheel }} + path: dist + + - name: Prepare + run: | + platform=${{ matrix.platform.docker }} + echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=semver,pattern={{version}} + type=sha + + - name: Set up QEMU + if: matrix.platform.docker != 'linux/amd64' + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push by digest + id: build + uses: docker/build-push-action@v6 + with: + context: . + platforms: ${{ matrix.platform.docker }} + labels: ${{ steps.meta.outputs.labels }} + tags: ${{ steps.meta.output.tags }} + outputs: type=image,name=${{ env.DOCKER_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.repository_owner == 'onekey-sec' && github.event_name == 'push' }} + + - name: Docker container vulnerability scan + id: scan + uses: anchore/scan-action@v6 + with: + image: ${{ env.DOCKER_IMAGE }} + fail-build: false + severity-cutoff: critical + only-fixed: true + + - name: Upload SARIF report + uses: github/codeql-action/upload-sarif@v3 + with: + sarif_file: ${{ steps.scan.outputs.sarif }} + + - name: Check unblob - help + run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --help + + - name: Check unblob - show-external-dependencies + run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --show-external-dependencies + + - name: Check unblob - run for a file with --verbose + run: docker run --rm -v "$(pwd)"/tests/integration/archive/zip/regular:/test ${{ env.DOCKER_IMAGE }}:latest -v -e /tmp /test/__input__/apple.zip + + - name: Export digest + run: | + mkdir -p /tmp/digests + digest="${{ steps.build.outputs.digest }}" + touch "/tmp/digests/${digest#sha256:}" + + - name: Upload digest + uses: actions/upload-artifact@v4 + with: + name: digest-${{ env.PLATFORM_PAIR }} + path: /tmp/digests/* + if-no-files-found: error + retention-days: 1 + + merge-and-push-image: + if: github.repository_owner == 'onekey-sec' && github.event_name == 'push' + runs-on: ubuntu-latest + needs: + - build-image + steps: + - name: Download digests + uses: actions/download-artifact@v4 + with: + path: /tmp/digests + pattern: digest-* + merge-multiple: true + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Docker meta + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.DOCKER_IMAGE }} + tags: | + type=raw,value=latest,enable={{is_default_branch}} + type=ref,event=branch + type=semver,pattern={{version}} + type=sha + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Create manifest list and push + working-directory: /tmp/digests + run: | + docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ + $(printf '${{ env.DOCKER_IMAGE }}@sha256:%s ' *) + - name: Inspect image + run: | + docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} + + release: + name: Release + runs-on: ubuntu-latest + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: + - build_linux_wheels + - build_macos_wheels + - build_sdist + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write + steps: + - uses: actions/download-artifact@v4 + with: + pattern: wheel-* + path: dist + merge-multiple: true + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: dist/* + - name: Publish to PyPI + if: ${{ startsWith(github.ref, 'refs/tags/') }} + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ if github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && secrets.TEST_PYPI_API_TOKEN || secrets.POETRY_PYPI_TOKEN_PYPI}} + with: + command: upload + args: --non-interactive --skip-existing dist/* ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.test_release && '--repository-url https://test.pypi.org || '' }} diff --git a/.github/workflows/build-publish-image.yml b/.github/workflows/build-publish-image.yml deleted file mode 100644 index fde30e41b5..0000000000 --- a/.github/workflows/build-publish-image.yml +++ /dev/null @@ -1,155 +0,0 @@ -name: Build Docker image - -on: - pull_request: - push: - branches: - - main - tags: - - '*' - -env: - DOCKER_IMAGE: ghcr.io/onekey-sec/unblob - -jobs: - build-image: - if: github.event_name == 'push' || contains(github.event.*.labels.*.name, 'dependencies') - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - platform: - - linux/amd64 - - linux/arm64 - steps: - - name: Prepare - run: | - platform=${{ matrix.platform }} - echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV - - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup git lfs - uses: ./.github/actions/setup-git-lfs - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - - - name: uv build - run: uv build --wheel - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKER_IMAGE }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch - type=semver,pattern={{version}} - type=sha - - - name: Set up QEMU - if: matrix.platform != 'linux/amd64' - uses: docker/setup-qemu-action@v3 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and push by digest - id: build - uses: docker/build-push-action@v6 - with: - context: . - platforms: ${{ matrix.platform }} - labels: ${{ steps.meta.outputs.labels }} - tags: ${{ steps.meta.output.tags }} - outputs: type=image,name=${{ env.DOCKER_IMAGE }},push-by-digest=true,name-canonical=true,push=${{ github.repository_owner == 'onekey-sec' && 'true' || 'false' }} - - - name: Docker container vulnerability scan - id: scan - uses: anchore/scan-action@v6 - with: - image: ${{ env.DOCKER_IMAGE }} - fail-build: false - severity-cutoff: critical - only-fixed: true - - - name: Upload SARIF report - uses: github/codeql-action/upload-sarif@v3 - with: - sarif_file: ${{ steps.scan.outputs.sarif }} - - - name: Check unblob - help - run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --help - - - name: Check unblob - show-external-dependencies - run: docker run --rm ${{ env.DOCKER_IMAGE }}:latest --show-external-dependencies - - - name: Check unblob - run for a file with --verbose - run: docker run --rm -v "$(pwd)"/tests/integration/archive/zip/regular:/test ${{ env.DOCKER_IMAGE }}:latest -v -e /tmp /test/__input__/apple.zip - - - name: Export digest - run: | - mkdir -p /tmp/digests - digest="${{ steps.build.outputs.digest }}" - touch "/tmp/digests/${digest#sha256:}" - - - name: Upload digest - uses: actions/upload-artifact@v4 - with: - name: digest-${{ env.PLATFORM_PAIR }} - path: /tmp/digests/* - if-no-files-found: error - retention-days: 1 - - merge-and-push-image: - if: github.repository_owner == 'onekey-sec' && github.event_name == 'push' - runs-on: ubuntu-latest - needs: - - build-image - steps: - - name: Download digests - uses: actions/download-artifact@v4 - with: - path: /tmp/digests - pattern: digest-* - merge-multiple: true - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - - name: Docker meta - id: meta - uses: docker/metadata-action@v5 - with: - images: ${{ env.DOCKER_IMAGE }} - tags: | - type=raw,value=latest,enable={{is_default_branch}} - type=ref,event=branch - type=semver,pattern={{version}} - type=sha - - - name: Login to GitHub Container Registry - uses: docker/login-action@v3 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Create manifest list and push - working-directory: /tmp/digests - run: | - docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \ - $(printf '${{ env.DOCKER_IMAGE }}@sha256:%s ' *) - - name: Inspect image - run: | - docker buildx imagetools inspect ${{ env.DOCKER_IMAGE }}:${{ steps.meta.outputs.version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index c4b5e52b13..0000000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: PyPI Release - -on: - release: - types: [created] - -jobs: - release: - runs-on: ubuntu-24.04 - - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup dependencies - uses: ./.github/actions/setup-dependencies - - - run: uv build - - run: uv publish - env: - UV_PUBLISH_TOKEN: ${{secrets.POETRY_PYPI_TOKEN_PYPI}} diff --git a/.github/workflows/run-tests.yml b/.github/workflows/run-tests.yml deleted file mode 100644 index cfdc674cd7..0000000000 --- a/.github/workflows/run-tests.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: Run checks and tests - -on: - push: - branches: - - main - pull_request: - merge_group: - -jobs: - check_pre_commit: - name: Check - pre-commit - runs-on: ubuntu-latest - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup Python - uses: actions/setup-python@v5 - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - - - name: Setup Nix - uses: cachix/install-nix-action@v30 - with: - install_url: https://releases.nixos.org/nix/nix-2.18.8/install - - - name: Check pre-commit hook - uses: pre-commit/action@v3.0.1 - - check_pyright: - name: Check - pyright - runs-on: ubuntu-latest - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - - - name: Check - pyright - run: uv run pyright . - - run_python_tests: - name: Run tests (Python) - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup 3rd party dependencies - uses: ./.github/actions/setup-dependencies - with: - python-version: ${{ matrix.python-version }} - - - name: Setup git lfs - uses: ./.github/actions/setup-git-lfs - - - name: Run pytest - run: uv run pytest -vvv - - run_rust_tests: - name: Run tests (Rust) - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: - - ubuntu-latest - - macos-latest - steps: - - name: Checkout source code - uses: actions/checkout@v4 - - - name: Setup sccache - uses: mozilla-actions/sccache-action@v0.0.7 - - - name: Run cargo test - run: cargo test diff --git a/pyproject.toml b/pyproject.toml index 2fc544d2f1..5619cb07b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,7 +37,7 @@ unblob = "unblob.cli:main" [dependency-groups] dev = [ - "atheris>=2.3,<3.0; python_version < '3.12'", + "atheris>=2.3,<3.0; sys_platform == 'linux' and python_version < '3.12'", "pre-commit>=3.5,<5.0", "pyright>=1.1.349", "pytest-cov>=3,<7", diff --git a/uv.lock b/uv.lock index fce05f38cc..af11566931 100644 --- a/uv.lock +++ b/uv.lock @@ -1517,7 +1517,7 @@ dependencies = [ [package.dev-dependencies] dev = [ - { name = "atheris", marker = "python_full_version < '3.12'" }, + { name = "atheris", marker = "python_full_version < '3.12' and sys_platform == 'linux'" }, { name = "pre-commit" }, { name = "pyright" }, { name = "pytest" }, @@ -1559,7 +1559,7 @@ requires-dist = [ [package.metadata.requires-dev] dev = [ - { name = "atheris", marker = "python_full_version < '3.12'", specifier = ">=2.3,<3.0" }, + { name = "atheris", marker = "python_full_version < '3.12' and sys_platform == 'linux'", specifier = ">=2.3,<3.0" }, { name = "pre-commit", specifier = ">=3.5,<5.0" }, { name = "pyright", specifier = ">=1.1.349" }, { name = "pytest", specifier = ">=8.0.0" },