From 6ddcac21e29dd41e9d2b6adba93e20fa590cd792 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Sat, 4 Mar 2023 08:35:15 -0700 Subject: [PATCH 01/40] aw shit, here we go again --- scripts/lib | 24 ++++++++++++++++++++++++ scripts/prisma-sed.sh | 4 ---- scripts/release/Dockerfile | 32 +++++++++++++++++++++++++------- scripts/release/build-utils.sh | 25 +++++++++++++++++++++++++ 4 files changed, 74 insertions(+), 11 deletions(-) create mode 100644 scripts/lib delete mode 100644 scripts/prisma-sed.sh create mode 100755 scripts/release/build-utils.sh diff --git a/scripts/lib b/scripts/lib new file mode 100644 index 000000000..ca75d1fc8 --- /dev/null +++ b/scripts/lib @@ -0,0 +1,24 @@ +#!/usr/bin/env bash + +create_dummy_rust_file() { + local path=$1 + mkdir -p $path/src + cp $path/**/Cargo.{toml,lock} $path/src + echo 'fn main() { println!("Wow, such empty!"); }' > $path/src/main.rs +} + +# prisma uses some `include_str!` macros that are mapped to locations on the host machine. so +# when we build in docker, we need to correct these paths according to the docker workdir. +# it's a bit of a hack, but it works lol +prisma_sed_correction() { + set -ex; \ + sed -i 's|\/.*\/core\/prisma\/schema.prisma|\/app\/core\/prisma\/schema.prisma|g' core/src/prisma.rs; \ + sed -i 's|\/.*\/core\/prisma\/migrations|\/app\/core\/prisma\/migrations|g' core/src/prisma.rs +} + +workspaces_sed_correction() { + set -ex; \ + sed -i '/core\/integration-tests/d' Cargo.toml; \ + sed -i '/apps\/desktop\/src-tauri/d' Cargo.toml; \ + sed -i '/apps\/tui/d' Cargo.toml +} \ No newline at end of file diff --git a/scripts/prisma-sed.sh b/scripts/prisma-sed.sh deleted file mode 100644 index 7590a9bff..000000000 --- a/scripts/prisma-sed.sh +++ /dev/null @@ -1,4 +0,0 @@ -# TODO: make more dyanimc of a script -set -ex; \ - sed -i 's|\/.*\/core\/prisma\/schema.prisma|\/app\/core\/prisma\/schema.prisma|g' core/src/prisma.rs; \ - sed -i 's|\/.*\/core\/prisma\/migrations|\/app\/core\/prisma\/migrations|g' core/src/prisma.rs \ No newline at end of file diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index b748fc02b..fff0160cd 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -75,17 +75,35 @@ WORKDIR /app ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +RUN rustup update && rustup target add x86_64-unknown-linux-musl + COPY .cargo .cargo -COPY . . +COPY ./Cargo.{toml,lock} . -RUN rustup update && rustup target add x86_64-unknown-linux-musl +# Run the build utils script to fix the prisma paths, remove unnecessary workspaces, and +# create the dummy rust files for core and server +RUN set -ex; \ + mkdir -p ./core/src; \ + mkdir -p ./server/src; \ + ./scripts/release/build-utils.sh -pwd 'core'; \ + ./scripts/release/build-utils.sh -d 'server' + +# Copy the core and server Cargo.toml and Cargo.lock files +COPY ./server/Cargo.{toml,lock} ./server/ +COPY ./core/Cargo.{toml,lock} ./core/ -# prisma uses some `include_str!` macros that are mapped to locations on the host machine. so -# when we build in docker, we need to correct these paths according to the docker workdir. -# it's a bit of a hack, but it works lol +# This is where the ~magic~ happens. We build the server (which pulls the core as a dependency) with +# the dummy files we created above. This ~should~ allow caching until the dependencies themselves change. RUN set -ex; \ - sed -i 's|\/.*\/core\/prisma\/schema.prisma|\/app\/core\/prisma\/schema.prisma|g' core/src/prisma.rs; \ - sed -i 's|\/.*\/core\/prisma\/migrations|\/app\/core\/prisma\/migrations|g' core/src/prisma.rs + cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ + rm -rf apps/server/src; \ + rm -rf apps/core/src + +# Now we can copy the real source files and build the server +COPY . . + +# TODO: I am not sure if the stump_core will be built correctly with this caching trick. I might just have to run a build for it... +# RUN cargo build --package stump_core --release --target x86_64-unknown-linux-musl ?? RUN cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl && \ cp target/x86_64-unknown-linux-musl/release/stump_server ./stump diff --git a/scripts/release/build-utils.sh b/scripts/release/build-utils.sh new file mode 100755 index 000000000..14a931b26 --- /dev/null +++ b/scripts/release/build-utils.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +SCRIPTS_DIR="${BASH_SOURCE%/*}/.." +source "${SCRIPTS_DIR}/lib" + +while getopts "pwd:" opt; do + case $opt in + p) + prisma_sed_correction + ;; + w) + workspaces_sed_correction + ;; + d) + path="$OPTARG" + echo "The path provided is $OPTARG" + create_dummy_rust_file $path + ;; + ?) + echo "Invalid option -$OPTARG" >&2 + exit 1 + ;; + esac +done +shift "$(($OPTIND -1))" \ No newline at end of file From d43d96ac2e2babdd0bfb77ddb7d0a7845ce9cd68 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:23:55 -0700 Subject: [PATCH 02/40] blindly testing --- .github/workflows/build_nix.yml | 4 +- .github/workflows/ci.yaml | 13 ++-- .github/workflows/nightly.yml | 132 ++++++++++++++++++++------------ scripts/release/Dockerfile | 14 ++-- scripts/release/build-docker.sh | 6 +- 5 files changed, 105 insertions(+), 64 deletions(-) diff --git a/.github/workflows/build_nix.yml b/.github/workflows/build_nix.yml index f09910002..77abb1e56 100644 --- a/.github/workflows/build_nix.yml +++ b/.github/workflows/build_nix.yml @@ -1,4 +1,4 @@ -name: "Build legacy Nix package on Ubuntu" +name: 'Build legacy Nix package on Ubuntu' on: push: @@ -15,5 +15,5 @@ jobs: - uses: cachix/install-nix-action@v17 - name: test run: nix develop --command "pkg-config" "--libs" "--cflags" "gdk-3.0" "gdk-3.0 >= 3.22" - # - name: Building package + # - name: Building package # run: nix develop --command pnpm core run setup && cargo check diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 07fc8a2ae..f58f2545e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,9 +9,7 @@ on: jobs: check-rust: name: Rust checks - runs-on: ubuntu-latest - # TODO: remove this once I have my own, hosted runner... - if: github.ref == 'refs/heads/main' + runs-on: [self-hosted] steps: - name: Checkout repository uses: actions/checkout@v3 @@ -26,13 +24,14 @@ jobs: run: | cargo fmt --all -- --check cargo clippy -- -D warnings - - name: Run tests - run: | - cargo integration-tests + # TODO: fix the tests, then uncomment this + # - name: Run tests + # run: | + # cargo integration-tests check-typescript: name: TypeScript checks - runs-on: ubuntu-latest + runs-on: [self-hosted] steps: - name: Checkout repository uses: actions/checkout@v3 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d83a24d59..4aa457f33 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,64 +1,102 @@ -name: Stump-nightly - -# on: -# schedule: -# # * is a special character in YAML so you have to quote this string -# - cron: '0 1 * * *' -# workflow_dispatch: +name: 'Stump nightly build and release' on: + # TODO: REMOVE THIS after testing... replace with push instead. pull_request: branches: - - 'nightly' + - develop + # push: + # branches: + # - develop jobs: - build-web: - strategy: - fail-fast: true - matrix: - plaform: [ubuntu-20.04] - name: Build web application - runs-on: ${{ matrix.plaform }} + # TODO: extract this into a reusable action that takes tags as input + docker-build: + name: Build docker image + runs-on: [self-hosted] steps: - name: Checkout repository uses: actions/checkout@v3 - - name: Build web - uses: ./.github/actions/build-web + - name: Get commit short sha + run: echo "::set-output name=GIT_REV::$(git rev-parse --short "$GITHUB_SHA")" - - name: Upload web build - uses: ./.github/actions/upload-artifact - with: - upload-name: web - upload-path: apps/web/dist - - build-server: - strategy: - fail-fast: true - matrix: - plaform: [ubuntu-20.04] # We should probably build for window and macos - name: Build server and desktop - runs-on: ${{ matrix.plaform }} - needs: build-web - steps: - - name: Checkout repository - uses: actions/checkout@v3 + - name: Setup rust + uses: ./.github/actions/setup-system + + - name: Generate Prisma client + uses: ./.github/actions/setup-prisma + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 - - name: Build server - uses: ./.github/actions/build-server + - name: Login to Docker Hub + uses: docker/login-action@v2 with: - platform: ${{ matrix.plaform }} + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Upload stump server - uses: ./.github/actions/upload-artifact + - name: Build and push + uses: docker/build-push-action@v4 with: - upload-name: stump_server - upload-path: target/release/stump_server + context: . + file: scripts/release/Dockerfile + # platforms: linux/arm64/v8,linux/amd64 + platforms: linux/amd64 + push: true + tags: aaronleopold/stump:nightly - - uses: tauri-apps/tauri-action@v0 + # TODO: uncomment once I resolve cross compilation with my single self-hosted runner + # build-web: + # strategy: + # fail-fast: true + # matrix: + # plaform: [ubuntu-20.04] + # name: Build web application + # runs-on: ${{ matrix.plaform }} + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 - - name: Upload desktop - uses: ./.github/actions/upload-artifact - with: - upload-name: stump-desktop - upload-path: target/release/bundle + # - name: Build web + # uses: ./.github/actions/build-web + + # - name: Upload web build + # uses: ./.github/actions/upload-artifact + # with: + # upload-name: web + # upload-path: apps/web/dist + + # build-server: + # strategy: + # fail-fast: true + # matrix: + # plaform: [ubuntu-20.04] # We should probably build for window and macos + # name: Build server and desktop + # runs-on: ${{ matrix.plaform }} + # needs: build-web + # steps: + # - name: Checkout repository + # uses: actions/checkout@v3 + + # - name: Build server + # uses: ./.github/actions/build-server + # with: + # platform: ${{ matrix.plaform }} + + # - name: Upload stump server + # uses: ./.github/actions/upload-artifact + # with: + # upload-name: stump_server + # upload-path: target/release/stump_server + + # - uses: tauri-apps/tauri-action@v0 + + # - name: Upload desktop + # uses: ./.github/actions/upload-artifact + # with: + # upload-name: stump-desktop + # upload-path: target/release/bundle diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index fff0160cd..d214d1244 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -80,12 +80,11 @@ RUN rustup update && rustup target add x86_64-unknown-linux-musl COPY .cargo .cargo COPY ./Cargo.{toml,lock} . -# Run the build utils script to fix the prisma paths, remove unnecessary workspaces, and -# create the dummy rust files for core and server +# Run the build utils script to create the dummy rust files for core and server RUN set -ex; \ mkdir -p ./core/src; \ mkdir -p ./server/src; \ - ./scripts/release/build-utils.sh -pwd 'core'; \ + ./scripts/release/build-utils.sh -d 'core'; \ ./scripts/release/build-utils.sh -d 'server' # Copy the core and server Cargo.toml and Cargo.lock files @@ -102,10 +101,11 @@ RUN set -ex; \ # Now we can copy the real source files and build the server COPY . . -# TODO: I am not sure if the stump_core will be built correctly with this caching trick. I might just have to run a build for it... -# RUN cargo build --package stump_core --release --target x86_64-unknown-linux-musl ?? - -RUN cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl && \ +RUN set -ex; \ + ./scripts/release/build-utils.sh -wp 'core'; \ + # TODO: determine if this is necessary (building core again) + cargo build --package stump_core --release --target x86_64-unknown-linux-musl; \ + cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ cp target/x86_64-unknown-linux-musl/release/stump_server ./stump ###################### diff --git a/scripts/release/build-docker.sh b/scripts/release/build-docker.sh index 2282f7f8c..177fc8879 100755 --- a/scripts/release/build-docker.sh +++ b/scripts/release/build-docker.sh @@ -1,7 +1,11 @@ #!/bin/bash FORMAT=${1:-auto} +PLATFORMS=${2:-linux/amd64} +TAG=${3:-latest} + GIT_REV=$(git rev-parse --short HEAD) set GIT_REV=$GIT_REV + # docker buildx build -f ./scripts/release/Dockerfile --push --platform=linux/arm64/v8,linux/amd64 -t aaronleopold/stump-preview:latest . -docker buildx build -f ./scripts/release/Dockerfile --push --progress=$FORMAT --platform=linux/amd64 -t aaronleopold/stump-preview:latest . \ No newline at end of file +docker buildx build -f ./scripts/release/Dockerfile --push --progress=$FORMAT --platform=$PLATFORMS -t aaronleopold/stump-preview:$TAG . \ No newline at end of file From ca637f41159ddb9e129af480f308a4ec0cd81ac2 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:45:39 -0700 Subject: [PATCH 03/40] ugh --- .github/workflows/nightly.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4aa457f33..899b7f455 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,10 +22,12 @@ jobs: run: echo "::set-output name=GIT_REV::$(git rev-parse --short "$GITHUB_SHA")" - name: Setup rust - uses: ./.github/actions/setup-system + # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: + # https://github.com/actions/runner/issues/1676 + uses: ${{ vars.RUNNER_WORKSPACE_PATH }}/.github/actions/setup-system - name: Generate Prisma client - uses: ./.github/actions/setup-prisma + uses: ${{ vars.RUNNER_WORKSPACE_PATH }}/.github/actions/setup-prisma - name: Set up QEMU uses: docker/setup-qemu-action@v2 From 0b751cbb988ebe93f838987f36e6d225f3123167 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:48:45 -0700 Subject: [PATCH 04/40] ugh x2 --- .github/workflows/ci.yaml | 2 +- .github/workflows/nightly.yml | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f58f2545e..a99769ea6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ name: Stump-CI on: - pull_request: + # pull_request: push: branches: - main diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 899b7f455..f5fcc099c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,10 +24,13 @@ jobs: - name: Setup rust # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: # https://github.com/actions/runner/issues/1676 - uses: ${{ vars.RUNNER_WORKSPACE_PATH }}/.github/actions/setup-system + working-directory: ${{ env.RUNNER_WORKSPACE_PATH }} + uses: ./.github/actions/setup-system - name: Generate Prisma client - uses: ${{ vars.RUNNER_WORKSPACE_PATH }}/.github/actions/setup-prisma + # TODO: kms + working-directory: ${{ env.RUNNER_WORKSPACE_PATH }} + uses: ./.github/actions/setup-prisma - name: Set up QEMU uses: docker/setup-qemu-action@v2 From 29d557cae0163f3c978aeec490d2981696295c5c Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:51:36 -0700 Subject: [PATCH 05/40] ugh x3 --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index f5fcc099c..8b5b0ae0d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,12 +24,12 @@ jobs: - name: Setup rust # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: # https://github.com/actions/runner/issues/1676 - working-directory: ${{ env.RUNNER_WORKSPACE_PATH }} + working-directory: ./stump uses: ./.github/actions/setup-system - name: Generate Prisma client # TODO: kms - working-directory: ${{ env.RUNNER_WORKSPACE_PATH }} + working-directory: ./stump uses: ./.github/actions/setup-prisma - name: Set up QEMU From 8772fa428a5f60185d49d9ec7a69021039bdfc92 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:53:06 -0700 Subject: [PATCH 06/40] could scream --- .github/workflows/nightly.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8b5b0ae0d..1f5d16e04 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,13 +24,10 @@ jobs: - name: Setup rust # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: # https://github.com/actions/runner/issues/1676 - working-directory: ./stump - uses: ./.github/actions/setup-system + uses: ./stump/.github/actions/setup-system - name: Generate Prisma client - # TODO: kms - working-directory: ./stump - uses: ./.github/actions/setup-prisma + uses: ./stump/.github/actions/setup-prisma - name: Set up QEMU uses: docker/setup-qemu-action@v2 From 13ec57f3c9432ae83091ccbbf985680b85afae17 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:54:32 -0700 Subject: [PATCH 07/40] :) --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1f5d16e04..e87d073e3 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,10 +24,10 @@ jobs: - name: Setup rust # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: # https://github.com/actions/runner/issues/1676 - uses: ./stump/.github/actions/setup-system + uses: ../.github/actions/setup-system - name: Generate Prisma client - uses: ./stump/.github/actions/setup-prisma + uses: ../.github/actions/setup-prisma - name: Set up QEMU uses: docker/setup-qemu-action@v2 From 78f5d54aa621e2fb3efdad74a415f2bc34a4d4d6 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:55:17 -0700 Subject: [PATCH 08/40] haha --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index e87d073e3..692b8ea83 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,10 +24,10 @@ jobs: - name: Setup rust # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: # https://github.com/actions/runner/issues/1676 - uses: ../.github/actions/setup-system + uses: ./../.github/actions/setup-system - name: Generate Prisma client - uses: ../.github/actions/setup-prisma + uses: ./../.github/actions/setup-prisma - name: Set up QEMU uses: docker/setup-qemu-action@v2 From 5fb873c4be26ae85373874aa3d65c3eb405551dc Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:57:20 -0700 Subject: [PATCH 09/40] ??? --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 692b8ea83..c41551370 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,10 +24,10 @@ jobs: - name: Setup rust # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: # https://github.com/actions/runner/issues/1676 - uses: ./../.github/actions/setup-system + uses: ./.github/actions/setup-system - name: Generate Prisma client - uses: ./../.github/actions/setup-prisma + uses: ./.github/actions/setup-prisma - name: Set up QEMU uses: docker/setup-qemu-action@v2 From 0ee21553b9b88762c40db528403f59c758543aae Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:58:26 -0700 Subject: [PATCH 10/40] HAHAHAHAHA --- .github/workflows/nightly.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c41551370..43d30cb18 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -22,9 +22,7 @@ jobs: run: echo "::set-output name=GIT_REV::$(git rev-parse --short "$GITHUB_SHA")" - name: Setup rust - # I have to use a variable prefixed by RUNNER_WORKSPACE_PATH because of this issue: - # https://github.com/actions/runner/issues/1676 - uses: ./.github/actions/setup-system + uses: ./.github/actions/setup-cargo - name: Generate Prisma client uses: ./.github/actions/setup-prisma From 6b9df1d4ec14941d6bab2e87c0e50452741b28c0 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Mon, 6 Mar 2023 17:59:48 -0700 Subject: [PATCH 11/40] stinky --- .github/actions/setup-prisma/action.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/actions/setup-prisma/action.yaml b/.github/actions/setup-prisma/action.yaml index 7e1470e33..e508dc73a 100644 --- a/.github/actions/setup-prisma/action.yaml +++ b/.github/actions/setup-prisma/action.yaml @@ -14,4 +14,4 @@ runs: working-directory: core if: steps.cache-prisma.outputs.cache-hit != 'true' shell: bash - run: cargo run -p prisma --release -- generate + run: cargo prisma generate From 490058ec73e6b18a60bb369da4baca775ab4d8b6 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:08:49 -0700 Subject: [PATCH 12/40] smellz --- .vscode/.todo | 1 + apps/server/src/routers/api/v1/log.rs | 1 - scripts/lib | 5 +-- scripts/release/Dockerfile | 50 ++++++++++++++++++++------- scripts/release/build-docker.sh | 2 +- 5 files changed, 42 insertions(+), 17 deletions(-) diff --git a/.vscode/.todo b/.vscode/.todo index ca4ef7813..19c4b60d6 100644 --- a/.vscode/.todo +++ b/.vscode/.todo @@ -76,3 +76,4 @@ - [ ] Make light mode look not disgusting lmao - [ ] ~Fix poor performance of EditLibraryModal~ - [ ] Just replace the modals with dedicated pages for editing and creating libraries +- [ ] `local-ip-address` doesn't compile to arm64.... need to either guard it or correct it. diff --git a/apps/server/src/routers/api/v1/log.rs b/apps/server/src/routers/api/v1/log.rs index b03997cfb..c516e2a0b 100644 --- a/apps/server/src/routers/api/v1/log.rs +++ b/apps/server/src/routers/api/v1/log.rs @@ -50,7 +50,6 @@ async fn get_logs() -> ApiResult<()> { )] /// Get information about the Stump log file, located at STUMP_CONFIG_DIR/Stump.log, or /// ~/.stump/Stump.log by default. Information such as the file size, last modified date, etc. -// #[get("/logs")] async fn get_logfile_info(session: ReadableSession) -> ApiResult> { get_session_admin_user(&session)?; let log_file_path = get_log_file(); diff --git a/scripts/lib b/scripts/lib index ca75d1fc8..ffd4c1c49 100644 --- a/scripts/lib +++ b/scripts/lib @@ -2,8 +2,8 @@ create_dummy_rust_file() { local path=$1 + echo "Creating dummy Rust file in $path" mkdir -p $path/src - cp $path/**/Cargo.{toml,lock} $path/src echo 'fn main() { println!("Wow, such empty!"); }' > $path/src/main.rs } @@ -20,5 +20,6 @@ workspaces_sed_correction() { set -ex; \ sed -i '/core\/integration-tests/d' Cargo.toml; \ sed -i '/apps\/desktop\/src-tauri/d' Cargo.toml; \ - sed -i '/apps\/tui/d' Cargo.toml + sed -i '/apps\/tui/d' Cargo.toml; \ + sed -i '/packages\/prisma-cli/d' Cargo.toml } \ No newline at end of file diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index d214d1244..d1fa2ff54 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -31,18 +31,39 @@ FROM messense/rust-musl-cross:aarch64-musl AS arm64-backend WORKDIR /app +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +RUN rustup target add aarch64-unknown-linux-musl + COPY .cargo .cargo -COPY . . +COPY scripts scripts +COPY Cargo.toml . -ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +# Run the build utils script to create the dummy rust files for core and server +RUN set -ex; \ + ./scripts/release/build-utils.sh -w; \ + mkdir -p ./core/src; \ + mkdir -p ./apps/server/src; \ + ./scripts/release/build-utils.sh -d 'core'; \ + ./scripts/release/build-utils.sh -d 'apps/server' -RUN rustup target add aarch64-unknown-linux-musl +COPY ./apps/server/Cargo.toml ./apps/server/ +COPY ./core/Cargo.toml ./core/ +# This is where the ~magic~ happens. We build the server (which pulls the core as a dependency) with +# the dummy files we created above. This ~should~ allow caching until the dependencies themselves change. RUN set -ex; \ - sed -i 's|\/.*\/core\/prisma\/schema.prisma|\/app\/core\/prisma\/schema.prisma|g' core/src/prisma.rs; \ - sed -i 's|\/.*\/core\/prisma\/migrations|\/app\/core\/prisma\/migrations|g' core/src/prisma.rs + cargo build --package stump_server --bin stump_server --release --target aarch64-unknown-linux-musl; \ + rm -rf ./apps/server/src; \ + rm -rf ./core/src + +# Now we can copy the real source files and build the server +COPY . . -RUN cargo build --package stump_server --bin stump_server --release --target aarch64-unknown-linux-musl && \ +RUN set -ex; \ + ./scripts/release/build-utils.sh -p 'core'; \ + # TODO: determine if this is necessary (building core again) + cargo build --package stump_core --release --target aarch64-unknown-linux-musl; \ + cargo build --package stump_server --bin stump_server --release --target aarch64-unknown-linux-musl; \ cp target/aarch64-unknown-linux-musl/release/stump_server ./stump ###################### @@ -77,32 +98,35 @@ ENV CARGO_NET_GIT_FETCH_WITH_CLI=true RUN rustup update && rustup target add x86_64-unknown-linux-musl +# TODO: make one-liner COPY .cargo .cargo -COPY ./Cargo.{toml,lock} . +COPY scripts scripts +COPY Cargo.toml . # Run the build utils script to create the dummy rust files for core and server RUN set -ex; \ + ./scripts/release/build-utils.sh -w; \ mkdir -p ./core/src; \ - mkdir -p ./server/src; \ + mkdir -p ./apps/server/src; \ ./scripts/release/build-utils.sh -d 'core'; \ - ./scripts/release/build-utils.sh -d 'server' + ./scripts/release/build-utils.sh -d 'apps/server' # Copy the core and server Cargo.toml and Cargo.lock files -COPY ./server/Cargo.{toml,lock} ./server/ -COPY ./core/Cargo.{toml,lock} ./core/ +COPY ./apps/server/Cargo.toml ./apps/server/ +COPY ./core/Cargo.toml ./core/ # This is where the ~magic~ happens. We build the server (which pulls the core as a dependency) with # the dummy files we created above. This ~should~ allow caching until the dependencies themselves change. RUN set -ex; \ cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ rm -rf apps/server/src; \ - rm -rf apps/core/src + rm -rf core/src # Now we can copy the real source files and build the server COPY . . RUN set -ex; \ - ./scripts/release/build-utils.sh -wp 'core'; \ + ./scripts/release/build-utils.sh -p 'core'; \ # TODO: determine if this is necessary (building core again) cargo build --package stump_core --release --target x86_64-unknown-linux-musl; \ cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ diff --git a/scripts/release/build-docker.sh b/scripts/release/build-docker.sh index 177fc8879..1ee4d9cbb 100755 --- a/scripts/release/build-docker.sh +++ b/scripts/release/build-docker.sh @@ -8,4 +8,4 @@ GIT_REV=$(git rev-parse --short HEAD) set GIT_REV=$GIT_REV # docker buildx build -f ./scripts/release/Dockerfile --push --platform=linux/arm64/v8,linux/amd64 -t aaronleopold/stump-preview:latest . -docker buildx build -f ./scripts/release/Dockerfile --push --progress=$FORMAT --platform=$PLATFORMS -t aaronleopold/stump-preview:$TAG . \ No newline at end of file +docker buildx build -f ./scripts/release/Dockerfile --load --progress=$FORMAT --platform=$PLATFORMS -t aaronleopold/stump-preview:$TAG . \ No newline at end of file From f0f6f31b6f235c298650cf986df26c49fbbe34b7 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:20:07 -0700 Subject: [PATCH 13/40] :angry: --- Cargo.lock | 3 +-- apps/server/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 24f7a3bf4..4beb62177 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2544,8 +2544,7 @@ dependencies = [ [[package]] name = "local-ip-address" version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "faa9d02443a1741e9f51dafdfcbffb3863b2a89c457d762b40337d6c5153ef81" +source = "git+https://github.com/EstebanBorai/local-ip-address.git?tag=v0.5.1#efb3a0b074ea9540bb055f1fe8f0fc5a5591b6bf" dependencies = [ "libc", "neli", diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index 9d8a0e4ef..cb5c3c18e 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -31,7 +31,7 @@ serde = { workspace = true } axum-sessions = "0.4.1" async-trait = "0.1.53" async-stream = { workspace = true } -local-ip-address = "0.5.1" +local-ip-address = "=0.5.1" ### Dev Utils ### From 8acb367d3590e89fb0312b52b23c1704fa81b896 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:38:52 -0700 Subject: [PATCH 14/40] hmmm --- .github/actions/setup-prisma/action.yaml | 4 ++-- apps/server/Cargo.toml | 5 ++++- scripts/release/Dockerfile | 1 + 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/actions/setup-prisma/action.yaml b/.github/actions/setup-prisma/action.yaml index e508dc73a..1f9d7ded9 100644 --- a/.github/actions/setup-prisma/action.yaml +++ b/.github/actions/setup-prisma/action.yaml @@ -11,7 +11,7 @@ runs: key: ${{ runner.os }}-prisma-${{ hashFiles('**/schema.prisma') }} - name: Generate Prisma client - working-directory: core + # working-directory: core if: steps.cache-prisma.outputs.cache-hit != 'true' shell: bash - run: cargo prisma generate + run: cargo prisma generate --schema=./core/prisma/schema.prisma diff --git a/apps/server/Cargo.toml b/apps/server/Cargo.toml index cb5c3c18e..b5174a2e6 100644 --- a/apps/server/Cargo.toml +++ b/apps/server/Cargo.toml @@ -31,7 +31,10 @@ serde = { workspace = true } axum-sessions = "0.4.1" async-trait = "0.1.53" async-stream = { workspace = true } -local-ip-address = "=0.5.1" +# TODO: figure out this super fucking annoying cargo dependency resolution issue. This is the second time +# cargo, in docker, has ignored the workspace version of this dep and instead used the latest version from crates.io +# local-ip-address = "0.5.1" +local-ip-address = { git = "https://github.com/EstebanBorai/local-ip-address.git", tag = "v0.5.1" } ### Dev Utils ### diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index d1fa2ff54..07c46ba8e 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -127,6 +127,7 @@ COPY . . RUN set -ex; \ ./scripts/release/build-utils.sh -p 'core'; \ + cat ./core/src/prisma.rs; \ # TODO: determine if this is necessary (building core again) cargo build --package stump_core --release --target x86_64-unknown-linux-musl; \ cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ From e22096d95314cd55cd42bd9a30af5c9d64210375 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Tue, 7 Mar 2023 16:46:54 -0700 Subject: [PATCH 15/40] oops --- scripts/release/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index 07c46ba8e..dff56d8d1 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -127,7 +127,7 @@ COPY . . RUN set -ex; \ ./scripts/release/build-utils.sh -p 'core'; \ - cat ./core/src/prisma.rs; \ + # cat ./core/src/prisma.rs; \ # TODO: determine if this is necessary (building core again) cargo build --package stump_core --release --target x86_64-unknown-linux-musl; \ cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ From 645015d655b44daf4f11172abf8c10a38ac2d773 Mon Sep 17 00:00:00 2001 From: Aaron Leopold Date: Tue, 7 Mar 2023 18:17:12 -0700 Subject: [PATCH 16/40] y no werk :sad: --- .dockerignore | 2 -- scripts/lib | 7 ++++++- scripts/release/Dockerfile | 18 +++++++----------- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/.dockerignore b/.dockerignore index 3ae4abc41..e707bc5fa 100644 --- a/.dockerignore +++ b/.dockerignore @@ -5,8 +5,6 @@ dist static apps/server/client target -*.lock -*-lock.* *.log *.db diff --git a/scripts/lib b/scripts/lib index ffd4c1c49..e3c8ae23c 100644 --- a/scripts/lib +++ b/scripts/lib @@ -4,7 +4,12 @@ create_dummy_rust_file() { local path=$1 echo "Creating dummy Rust file in $path" mkdir -p $path/src - echo 'fn main() { println!("Wow, such empty!"); }' > $path/src/main.rs + if [[ $path == "core" ]]; then + echo 'fn foo() { println!("Wow, such empty!"); }' > $path/src/lib.rs + else + echo 'fn main() { println!("Wow, such empty!"); }' > $path/src/main.rs + fi + } # prisma uses some `include_str!` macros that are mapped to locations on the host machine. so diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index dff56d8d1..f4b14498f 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -101,35 +101,31 @@ RUN rustup update && rustup target add x86_64-unknown-linux-musl # TODO: make one-liner COPY .cargo .cargo COPY scripts scripts -COPY Cargo.toml . +COPY Cargo.toml Cargo.lock ./ # Run the build utils script to create the dummy rust files for core and server RUN set -ex; \ ./scripts/release/build-utils.sh -w; \ - mkdir -p ./core/src; \ - mkdir -p ./apps/server/src; \ - ./scripts/release/build-utils.sh -d 'core'; \ - ./scripts/release/build-utils.sh -d 'apps/server' + ./scripts/release/build-utils.sh -d './core'; \ + ./scripts/release/build-utils.sh -d './apps/server' -# Copy the core and server Cargo.toml and Cargo.lock files +# Copy the core and server Cargo.{toml,lock} files COPY ./apps/server/Cargo.toml ./apps/server/ COPY ./core/Cargo.toml ./core/ # This is where the ~magic~ happens. We build the server (which pulls the core as a dependency) with # the dummy files we created above. This ~should~ allow caching until the dependencies themselves change. RUN set -ex; \ - cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ + cargo build --release --target x86_64-unknown-linux-musl; \ rm -rf apps/server/src; \ rm -rf core/src # Now we can copy the real source files and build the server COPY . . +COPY ./core/src/prisma.rs ./core/src/prisma.rs RUN set -ex; \ - ./scripts/release/build-utils.sh -p 'core'; \ - # cat ./core/src/prisma.rs; \ - # TODO: determine if this is necessary (building core again) - cargo build --package stump_core --release --target x86_64-unknown-linux-musl; \ + ./scripts/release/build-utils.sh -p; \ cargo build --package stump_server --bin stump_server --release --target x86_64-unknown-linux-musl; \ cp target/x86_64-unknown-linux-musl/release/stump_server ./stump From 8a6895da18b963aff4e97c040841f8d293a4b836 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Tue, 7 Mar 2023 18:33:12 -0700 Subject: [PATCH 17/40] testing arm --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 43d30cb18..8ba60f7c0 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -44,8 +44,8 @@ jobs: with: context: . file: scripts/release/Dockerfile - # platforms: linux/arm64/v8,linux/amd64 - platforms: linux/amd64 + platforms: linux/arm64/v8,linux/amd64 + # platforms: linux/amd64 push: true tags: aaronleopold/stump:nightly From c56f25c4b0923797a92e929370efa9c5663f9dd5 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Tue, 7 Mar 2023 18:37:24 -0700 Subject: [PATCH 18/40] testing arm oops --- scripts/release/Dockerfile | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index f4b14498f..21fafcc5d 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -34,35 +34,34 @@ WORKDIR /app ENV CARGO_NET_GIT_FETCH_WITH_CLI=true RUN rustup target add aarch64-unknown-linux-musl +# TODO: make one-liner COPY .cargo .cargo COPY scripts scripts -COPY Cargo.toml . +COPY Cargo.toml Cargo.lock ./ # Run the build utils script to create the dummy rust files for core and server RUN set -ex; \ ./scripts/release/build-utils.sh -w; \ - mkdir -p ./core/src; \ - mkdir -p ./apps/server/src; \ - ./scripts/release/build-utils.sh -d 'core'; \ - ./scripts/release/build-utils.sh -d 'apps/server' + ./scripts/release/build-utils.sh -d './core'; \ + ./scripts/release/build-utils.sh -d './apps/server' +# Copy the core and server Cargo.{toml,lock} files COPY ./apps/server/Cargo.toml ./apps/server/ COPY ./core/Cargo.toml ./core/ # This is where the ~magic~ happens. We build the server (which pulls the core as a dependency) with # the dummy files we created above. This ~should~ allow caching until the dependencies themselves change. RUN set -ex; \ - cargo build --package stump_server --bin stump_server --release --target aarch64-unknown-linux-musl; \ - rm -rf ./apps/server/src; \ - rm -rf ./core/src + cargo build --release --target aarch64-unknown-linux-musl; \ + rm -rf apps/server/src; \ + rm -rf core/src # Now we can copy the real source files and build the server COPY . . +COPY ./core/src/prisma.rs ./core/src/prisma.rs RUN set -ex; \ - ./scripts/release/build-utils.sh -p 'core'; \ - # TODO: determine if this is necessary (building core again) - cargo build --package stump_core --release --target aarch64-unknown-linux-musl; \ + ./scripts/release/build-utils.sh -p; \ cargo build --package stump_server --bin stump_server --release --target aarch64-unknown-linux-musl; \ cp target/aarch64-unknown-linux-musl/release/stump_server ./stump From 855a3179d7ee449ebc5ce558be9a30c714a35a82 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Wed, 8 Mar 2023 17:32:44 -0700 Subject: [PATCH 19/40] tweaks --- .github/actions/setup-cargo/action.yml | 6 ++++ .github/workflows/nightly.yml | 24 ++++++++------ scripts/release/Dockerfile | 44 +++++++++++++++++++++++--- scripts/release/build-docker.sh | 7 ++-- 4 files changed, 63 insertions(+), 18 deletions(-) diff --git a/.github/actions/setup-cargo/action.yml b/.github/actions/setup-cargo/action.yml index b67987b2e..ebcb1300d 100644 --- a/.github/actions/setup-cargo/action.yml +++ b/.github/actions/setup-cargo/action.yml @@ -5,6 +5,12 @@ runs: using: 'composite' steps: - name: System setup + # TODO: determine if this is actually OK... I keep getting locked out of sudo + # because the setup script has a few sudo commands that fail. I either need to setup + # some sort of askpass or (ideally) not run setup if it isn't needed on the self-hosted runner... + # Only run if we're NOT on a self-hosted runner. This check is really naive and should + # be changed. I doubt I'll host more than one runner, so this is fine for now. + if: runner.name != 'manjaro-az' shell: bash run: CHECK_NODE=0 CHECK_CARGO=0 DEV_SETUP=0 ./scripts/system-setup.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8ba60f7c0..fee96b068 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,13 +1,12 @@ name: 'Stump nightly build and release' on: - # TODO: REMOVE THIS after testing... replace with push instead. pull_request: branches: - develop - # push: - # branches: - # - develop + push: + branches: + - develop jobs: # TODO: extract this into a reusable action that takes tags as input @@ -19,7 +18,8 @@ jobs: uses: actions/checkout@v3 - name: Get commit short sha - run: echo "::set-output name=GIT_REV::$(git rev-parse --short "$GITHUB_SHA")" + # run: echo "::set-output name=GIT_REV::$(git rev-parse --short "$GITHUB_SHA")" + run: echo "GIT_REV=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV - name: Setup rust uses: ./.github/actions/setup-cargo @@ -39,14 +39,20 @@ jobs: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Build and push + - name: Run buildx build uses: docker/build-push-action@v4 with: context: . + build-args: | + "GIT_REV=${{ env.GIT_REV }}" file: scripts/release/Dockerfile - platforms: linux/arm64/v8,linux/amd64 - # platforms: linux/amd64 - push: true + platforms: ${{ vars.SUPPORTED_DOCKER_PLATFORMS}} + # When a PR is opened against develop, we want to build the image but not push it. + load: github.event_name == 'pull_request' + # When a PR is merged into develop, we want to build and push the image. Note that + # unless there was a push without a PR, much of the docker build will come out of + # the cache. + push: github.event_name != 'pull_request' tags: aaronleopold/stump:nightly # TODO: uncomment once I resolve cross compilation with my single self-hosted runner diff --git a/scripts/release/Dockerfile b/scripts/release/Dockerfile index 21fafcc5d..9635a3758 100644 --- a/scripts/release/Dockerfile +++ b/scripts/release/Dockerfile @@ -29,6 +29,9 @@ RUN mv ./apps/web/dist build FROM messense/rust-musl-cross:aarch64-musl AS arm64-backend +ARG GIT_REV +ENV GIT_REV=${GIT_REV} + WORKDIR /app ENV CARGO_NET_GIT_FETCH_WITH_CLI=true @@ -65,6 +68,7 @@ RUN set -ex; \ cargo build --package stump_server --bin stump_server --release --target aarch64-unknown-linux-musl; \ cp target/aarch64-unknown-linux-musl/release/stump_server ./stump +# FIXME: armv7 is currently broken. I have a gut it needs a similar workaround as the arm64 ###################### ### armv7 / arm/v7 ### ###################### @@ -73,16 +77,44 @@ RUN set -ex; \ # out how to have the name be v7 inclusive so FROM messense/rust-musl-cross:armv7-musleabihf@sha256:3e133558686fd5059ce25749cece40a81d87dad2c7a68727c36a1bcacba6752c AS arm-backend +ARG GIT_REV +ENV GIT_REV=${GIT_REV} + WORKDIR /app +ENV CARGO_NET_GIT_FETCH_WITH_CLI=true + +RUN rustup update && rustup target add armv7-unknown-linux-musleabihf + +# TODO: make one-liner COPY .cargo .cargo -COPY . . +COPY scripts scripts +COPY Cargo.toml Cargo.lock ./ -ENV CARGO_NET_GIT_FETCH_WITH_CLI=true +# Run the build utils script to create the dummy rust files for core and server +RUN set -ex; \ + ./scripts/release/build-utils.sh -w; \ + ./scripts/release/build-utils.sh -d './core'; \ + ./scripts/release/build-utils.sh -d './apps/server' + +# Copy the core and server Cargo.{toml,lock} files +COPY ./apps/server/Cargo.toml ./apps/server/ +COPY ./core/Cargo.toml ./core/ + +# This is where the ~magic~ happens. We build the server (which pulls the core as a dependency) with +# the dummy files we created above. This ~should~ allow caching until the dependencies themselves change. +RUN set -ex; \ + cargo build --release --target armv7-unknown-linux-musleabihf; \ + rm -rf apps/server/src; \ + rm -rf core/src -RUN rustup target add armv7-unknown-linux-musleabihf +# Now we can copy the real source files and build the server +COPY . . +COPY ./core/src/prisma.rs ./core/src/prisma.rs -RUN cargo build --package stump_server --bin stump_server --release --target armv7-unknown-linux-musleabihf && \ +RUN set -ex; \ + ./scripts/release/build-utils.sh -p; \ + cargo build --package stump_server --bin stump_server --release --target armv7-unknown-linux-musleabihf; \ cp target/armv7-unknown-linux-musleabihf/release/stump_server ./stump ###################### @@ -91,6 +123,10 @@ RUN cargo build --package stump_server --bin stump_server --release --target arm FROM messense/rust-musl-cross:x86_64-musl AS amd64-backend +ARG GIT_REV +ENV GIT_REV=${GIT_REV} + + WORKDIR /app ENV CARGO_NET_GIT_FETCH_WITH_CLI=true diff --git a/scripts/release/build-docker.sh b/scripts/release/build-docker.sh index 1ee4d9cbb..4ab3651f6 100755 --- a/scripts/release/build-docker.sh +++ b/scripts/release/build-docker.sh @@ -2,10 +2,7 @@ FORMAT=${1:-auto} PLATFORMS=${2:-linux/amd64} -TAG=${3:-latest} - +TAG=${3:-nightly} GIT_REV=$(git rev-parse --short HEAD) -set GIT_REV=$GIT_REV -# docker buildx build -f ./scripts/release/Dockerfile --push --platform=linux/arm64/v8,linux/amd64 -t aaronleopold/stump-preview:latest . -docker buildx build -f ./scripts/release/Dockerfile --load --progress=$FORMAT --platform=$PLATFORMS -t aaronleopold/stump-preview:$TAG . \ No newline at end of file +docker buildx build -f ./scripts/release/Dockerfile --load --progress=$FORMAT --platform=$PLATFORMS -t aaronleopold/stump:$TAG --build-arg GIT_REV=$GIT_REV . \ No newline at end of file From ff2e3b2c54995d7f5a1a42c8943d3ef718e04ff3 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Wed, 8 Mar 2023 17:38:40 -0700 Subject: [PATCH 20/40] tweaks --- .github/workflows/nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fee96b068..2d4b067ec 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -48,11 +48,11 @@ jobs: file: scripts/release/Dockerfile platforms: ${{ vars.SUPPORTED_DOCKER_PLATFORMS}} # When a PR is opened against develop, we want to build the image but not push it. - load: github.event_name == 'pull_request' + load: ${{ github.event_name == 'pull_request' }} # When a PR is merged into develop, we want to build and push the image. Note that # unless there was a push without a PR, much of the docker build will come out of # the cache. - push: github.event_name != 'pull_request' + push: ${{ github.event_name == 'push' }} tags: aaronleopold/stump:nightly # TODO: uncomment once I resolve cross compilation with my single self-hosted runner From d6f291932347fd0647ea8376821cc7b93ae8c2e6 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Wed, 8 Mar 2023 17:50:32 -0700 Subject: [PATCH 21/40] fix conditions --- .github/workflows/nightly.yml | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 2d4b067ec..dbbd18484 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -27,8 +27,22 @@ jobs: - name: Generate Prisma client uses: ./.github/actions/setup-prisma + - name: Setup buildx platforms + # if github.event_name == 'pull_request', platform is only linux/amd64. Otherwise, we + # build for the existing vars.SUPPORTED_DOCKER_PLATFORMS + run: | + if [ "${{ github.event_name }}" == "pull_request" ]; then + echo "SUPPORTED_DOCKER_PLATFORMS=linux/amd64" >> $GITHUB_ENV + echo "SETUP_QEMU=0" >> $GITHUB_ENV + else + echo "SUPPORTED_DOCKER_PLATFORMS=${{ vars.SUPPORTED_DOCKER_PLATFORMS }}" >> $GITHUB_ENV + echo "SETUP_QEMU=1" >> $GITHUB_ENV + fi + - name: Set up QEMU uses: docker/setup-qemu-action@v2 + # Only setup qemu if we're not only building for linux/amd64 + if: ${{ env.SETUP_QEMU == 1 }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -46,7 +60,7 @@ jobs: build-args: | "GIT_REV=${{ env.GIT_REV }}" file: scripts/release/Dockerfile - platforms: ${{ vars.SUPPORTED_DOCKER_PLATFORMS}} + platforms: ${{ env.SUPPORTED_DOCKER_PLATFORMS }} # When a PR is opened against develop, we want to build the image but not push it. load: ${{ github.event_name == 'pull_request' }} # When a PR is merged into develop, we want to build and push the image. Note that From 979accb9f0d2171ebeba58a14ee8402843ecbf7a Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Wed, 8 Mar 2023 18:00:08 -0700 Subject: [PATCH 22/40] test cache --- apps/server/src/main.rs | 2 ++ core/src/lib.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index 548ae982b..b0e15e776 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -71,5 +71,7 @@ async fn main() -> ServerResult<()> { .await .expect("Failed to start Stump HTTP server!"); + println!("Simulate server codebase change..."); + Ok(()) } diff --git a/core/src/lib.rs b/core/src/lib.rs index d0c7c3602..5a415c021 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -52,6 +52,7 @@ impl StumpCore { let core_ctx = Ctx::new(internal_channel.0).await; let event_manager = EventManager::new(core_ctx.get_ctx(), internal_channel.1); + println!("Simulate core codebase change..."); StumpCore { ctx: core_ctx, From 6051ad3238ca46f2b50861799e01a789badf54bc Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Wed, 8 Mar 2023 18:08:57 -0700 Subject: [PATCH 23/40] try fix cache --- .github/workflows/nightly.yml | 17 +++++++++++++++++ apps/server/src/main.rs | 2 -- core/src/lib.rs | 1 - 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index dbbd18484..a8ecca8dd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -39,6 +39,14 @@ jobs: echo "SETUP_QEMU=1" >> $GITHUB_ENV fi + - name: Setup Docker layers cache + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx- + restore-keys: | + ${{ runner.os }}-buildx- + - name: Set up QEMU uses: docker/setup-qemu-action@v2 # Only setup qemu if we're not only building for linux/amd64 @@ -68,6 +76,15 @@ jobs: # the cache. push: ${{ github.event_name == 'push' }} tags: aaronleopold/stump:nightly + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + # https://github.com/docker/build-push-action/issues/252 + # TODO: https://github.com/moby/buildkit/issues/1896 + - name: Move buildx cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache # TODO: uncomment once I resolve cross compilation with my single self-hosted runner # build-web: diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index b0e15e776..548ae982b 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -71,7 +71,5 @@ async fn main() -> ServerResult<()> { .await .expect("Failed to start Stump HTTP server!"); - println!("Simulate server codebase change..."); - Ok(()) } diff --git a/core/src/lib.rs b/core/src/lib.rs index 5a415c021..d0c7c3602 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -52,7 +52,6 @@ impl StumpCore { let core_ctx = Ctx::new(internal_channel.0).await; let event_manager = EventManager::new(core_ctx.get_ctx(), internal_channel.1); - println!("Simulate core codebase change..."); StumpCore { ctx: core_ctx, From b867cac51d14eac9341317c51eb53a7aeb775176 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Wed, 8 Mar 2023 18:26:05 -0700 Subject: [PATCH 24/40] try fix cache again --- .github/workflows/nightly.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index a8ecca8dd..07dcee9a7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -43,7 +43,7 @@ jobs: uses: actions/cache@v3 with: path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx- + key: ${{ runner.os }}-buildx-${{ github.sha }} restore-keys: | ${{ runner.os }}-buildx- From adc0b559b39fd9547a94e7bf012f0f81c892436f Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Wed, 8 Mar 2023 19:06:35 -0700 Subject: [PATCH 25/40] test cache again --- apps/server/src/main.rs | 2 ++ core/src/lib.rs | 1 + 2 files changed, 3 insertions(+) diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index 548ae982b..b0e15e776 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -71,5 +71,7 @@ async fn main() -> ServerResult<()> { .await .expect("Failed to start Stump HTTP server!"); + println!("Simulate server codebase change..."); + Ok(()) } diff --git a/core/src/lib.rs b/core/src/lib.rs index d0c7c3602..5a415c021 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -52,6 +52,7 @@ impl StumpCore { let core_ctx = Ctx::new(internal_channel.0).await; let event_manager = EventManager::new(core_ctx.get_ctx(), internal_channel.1); + println!("Simulate core codebase change..."); StumpCore { ctx: core_ctx, From 1967a34064a97dd4bc86140d3658efe0fdb5ccc7 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:47:37 -0700 Subject: [PATCH 26/40] test just core change --- core/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/core/src/lib.rs b/core/src/lib.rs index 5a415c021..d0c7c3602 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -52,7 +52,6 @@ impl StumpCore { let core_ctx = Ctx::new(internal_channel.0).await; let event_manager = EventManager::new(core_ctx.get_ctx(), internal_channel.1); - println!("Simulate core codebase change..."); StumpCore { ctx: core_ctx, From 2842dbc93d2477ac69239fd98f2f332f5fb688c7 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:51:52 -0700 Subject: [PATCH 27/40] try ignore --- .dockerignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.dockerignore b/.dockerignore index e707bc5fa..7618ef07d 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,7 @@ apps/server/client target *.log *.db +.git # ignore contents in excess directories, some are kept only so cargo # doesn't yell at me From 7d7de3c0d4c118567713ffaaea478680402124b2 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Thu, 9 Mar 2023 18:17:27 -0700 Subject: [PATCH 28/40] pause on caching ig :sob: --- .github/workflows/ci.yaml | 32 +------------------------------- apps/server/src/main.rs | 2 -- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a99769ea6..df425e27e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,7 +1,7 @@ name: Stump-CI on: - # pull_request: + pull_request: push: branches: - main @@ -44,33 +44,3 @@ jobs: # - name: typecheck # run: pnpm moon run :typecheck - - # release: - # name: Release (${{ matrix.platform }}) - # runs-on: ${{ matrix.platform }} - # # Release only runs on push to main. TODO: can I make this trigger on tag? - # if: github.event_name != 'pull_request' - # strategy: - # fail-fast: true - # matrix: - # platform: [ubuntu-latest, macos-latest, windows-latest] - # steps: - # - name: Checkout repository - # uses: actions/checkout@v3 - - # - name: Install Rust - # uses: actions-rs/toolchain@v1 - # with: - # toolchain: stable - # profile: minimal - # override: true - # # TODO: clippy?? - # components: rustfmt, rust-src - - # # TODO: figure out caching for rust deps - - # - name: Generate Prisma client - # uses: ./.github/actions/generate-prisma-client - - # TODO: pnpm setup - # TODO: docker builds -> maybe this helps? https://github.com/devture/matrix-corporal/blob/master/.gitlab-ci.yml diff --git a/apps/server/src/main.rs b/apps/server/src/main.rs index b0e15e776..548ae982b 100644 --- a/apps/server/src/main.rs +++ b/apps/server/src/main.rs @@ -71,7 +71,5 @@ async fn main() -> ServerResult<()> { .await .expect("Failed to start Stump HTTP server!"); - println!("Simulate server codebase change..."); - Ok(()) } From 1e81dc531132aedc642e7cde2b336672a2788b10 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Thu, 9 Mar 2023 18:20:13 -0700 Subject: [PATCH 29/40] add cancelling? --- .github/workflows/ci.yaml | 4 ++++ .github/workflows/nightly.yml | 6 +++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index df425e27e..66817d65b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,6 +6,10 @@ on: branches: - main +concurrency: + group: ${{ github.head_ref }} + cancel-in-progress: true + jobs: check-rust: name: Rust checks diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 07dcee9a7..1724a622d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,4 +1,4 @@ -name: 'Stump nightly build and release' +name: 'Stump nightly build' on: pull_request: @@ -8,6 +8,10 @@ on: branches: - develop +concurrency: + group: ${{ github.head_ref }} + cancel-in-progress: true + jobs: # TODO: extract this into a reusable action that takes tags as input docker-build: From ec4e3c429ac030a7c622f1f869ffee0e855f673a Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Thu, 9 Mar 2023 18:45:53 -0700 Subject: [PATCH 30/40] hrm --- .gitignore | 3 ++- apps/web/dist/.placeholder | 1 + scripts/system-setup.sh | 8 +++++--- 3 files changed, 8 insertions(+), 4 deletions(-) create mode 100644 apps/web/dist/.placeholder diff --git a/.gitignore b/.gitignore index 6a933ccdd..f08f63794 100644 --- a/.gitignore +++ b/.gitignore @@ -18,7 +18,8 @@ logs/ build/ coverage/ cjs/ -dist/ +**/*/dist/**/* +!**/*/dist/.placeholder dts/ esm/ lib/ diff --git a/apps/web/dist/.placeholder b/apps/web/dist/.placeholder new file mode 100644 index 000000000..32f95c0d1 --- /dev/null +++ b/apps/web/dist/.placeholder @@ -0,0 +1 @@ +hi \ No newline at end of file diff --git a/scripts/system-setup.sh b/scripts/system-setup.sh index ff1c4820d..5086d7d5e 100755 --- a/scripts/system-setup.sh +++ b/scripts/system-setup.sh @@ -88,16 +88,18 @@ if [ ${_CHECK_NODE} == 1 ]; then fi fi +# TODO: group these? +# https://tauri.app/v1/guides/getting-started/prerequisites/#1-system-dependencies if [[ "$OSTYPE" == "linux-gnu"* ]]; then if which apt-get &> /dev/null; then sudo apt-get -y update - sudo apt-get -y install pkg-config libssl-dev libdbus-1-dev libsoup2.4-dev libwebkit2gtk-4.0-dev curl wget libgtk-3-dev libappindicator3-dev librsvg2-dev build-essential + sudo apt-get -y install pkg-config libssl-dev libdbus-1-dev libsoup2.4-dev libwebkit2gtk-4.0-dev curl wget libgtk-3-dev libappindicator3-dev librsvg2-dev build-essential libayatana-appindicator3-dev elif which pacman &> /dev/null; then sudo pacman -Syu - sudo pacman -S --needed base-devel openssl + sudo pacman -S --needed webkit2gtk base-devel curl wget openssl appmenu-gtk-module gtk3 libappindicator-gtk3 librsvg libvips elif which dnf &> /dev/null; then sudo dnf check-update - sudo dnf install "openssl-devel" + sudo dnf install openssl-devel webkit2gtk4.0-devel curl wget libappindicator-gtk3 librsvg2-devel sudo dnf group install "C Development Tools and Libraries" else echo "Your distro '$(lsb_release -s -d)' is not supported by this script. Please consider adding support for it: https://github.com/aaronleopold/stump/issues" From dab7e79aed0a7d46350d1df5137ce3a4c589048e Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Thu, 9 Mar 2023 18:49:34 -0700 Subject: [PATCH 31/40] ugh, figure something else out for cancelling... --- .github/workflows/ci.yaml | 4 ---- .github/workflows/nightly.yml | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 66817d65b..df425e27e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,10 +6,6 @@ on: branches: - main -concurrency: - group: ${{ github.head_ref }} - cancel-in-progress: true - jobs: check-rust: name: Rust checks diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1724a622d..4483d88bf 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -8,10 +8,6 @@ on: branches: - develop -concurrency: - group: ${{ github.head_ref }} - cancel-in-progress: true - jobs: # TODO: extract this into a reusable action that takes tags as input docker-build: From 1ae9b81de9be41634195c89a84e277e24e678491 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:13:33 -0700 Subject: [PATCH 32/40] push up yesterdays stuff oops --- .github/actions/build-desktop/action.yml | 37 ++++ .github/actions/build-docker/action.yml | 87 ++++++++ .github/actions/build-server/action.yml | 29 +-- .github/actions/build-web/action.yml | 27 ++- .github/actions/compile-web/action.yml | 20 -- .github/actions/setup-cargo/action.yml | 15 +- .github/actions/upload-artifact/action.yml | 1 - .github/workflows/ci.yaml | 5 +- .github/workflows/nightly.yml | 222 ++++++++++++--------- .vscode/.todo | 3 +- Cargo.toml | 2 +- core/src/db/models/log.rs | 9 +- core/src/prelude/enums.rs | 9 +- core/src/prelude/server/query.rs | 9 +- scripts/lib | 5 + scripts/release/compile-server.sh | 80 ++++++++ scripts/system-setup.sh | 39 ++-- 17 files changed, 407 insertions(+), 192 deletions(-) create mode 100644 .github/actions/build-desktop/action.yml create mode 100644 .github/actions/build-docker/action.yml delete mode 100644 .github/actions/compile-web/action.yml create mode 100755 scripts/release/compile-server.sh diff --git a/.github/actions/build-desktop/action.yml b/.github/actions/build-desktop/action.yml new file mode 100644 index 000000000..615d3cb8e --- /dev/null +++ b/.github/actions/build-desktop/action.yml @@ -0,0 +1,37 @@ +name: 'Build Stump desktop app' +description: 'Compile the Stump desktop app' + +inputs: + platform: + description: 'The plaform of the runner' + required: true + +runs: + using: composite + steps: + - name: Checkout project + uses: actions/checkout@v3 + + # - name: Configure environment + # run: | + # if [[ ${{ inputs.platform }} == 'linux' || ${{ inputs.platform }} == 'windows' ]]; then + # echo "RUN_SETUP=false" >> $GITHUB_ENV + # else + # echo "RUN_SETUP=true" >> $GITHUB_ENV + # fi + + - name: Setup rust + uses: ./.github/actions/setup-cargo + + - name: Generate Prisma client + uses: ./.github/actions/setup-prisma + + - name: Copy bundled web app + uses: actions/download-artifact@v3 + with: + name: webapp + path: ./apps/desktop/dist + + - name: Compile desktop app + shell: bash + run: cargo build --package stump_desktop --release diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml new file mode 100644 index 000000000..e92a5d189 --- /dev/null +++ b/.github/actions/build-docker/action.yml @@ -0,0 +1,87 @@ +name: 'Build docker image' +description: 'Build and load or push a tagged docker image for stump' + +inputs: + load: + description: 'Set output-type to docker' + default: true + push: + description: 'Set output-type to registry' + default: false + tags: + description: 'List of tags to assigned to the image' + default: 'nightly' + platforms: + description: 'List of platforms to build' + required: true + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Get commit short sha + run: echo "GIT_REV=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV + + - name: Format tags + run: | + echo "TAGS=$(echo ${{ inputs.tags }} | sed -e 's/,/,aaronleopold\/stump:/g' | sed -e 's/^/aaronleopold\/stump:/')" >> $GITHUB_ENV + + - name: Setup rust + uses: ./.github/actions/setup-cargo + + - name: Generate Prisma client + uses: ./.github/actions/setup-prisma + + - name: Setup Docker layers cache + uses: actions/cache@v3 + with: + path: /tmp/.buildx-cache + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + + # We only need QEMU when an arm* platform is targeted + - name: Check QEMU requirement + id: check-qemu + run: | + if [[ ${{ inputs.platforms }} == *"arm"* ]]; then + echo "SETUP_QEMU=1" >> $GITHUB_OUTPUT + else + echo "SETUP_QEMU=0" >> $GITHUB_OUTPUT + fi + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + if: ${{ steps.check-qemu.outputs.SETUP_QEMU == '1' }} + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Login to Docker Hub + uses: docker/login-action@v2 + with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + + - name: Run buildx build + uses: docker/build-push-action@v4 + with: + context: . + build-args: | + "GIT_REV=${{ env.GIT_REV }}" + file: scripts/release/Dockerfile + platforms: ${{ inputs.platforms }} + load: ${{ inputs.load }} + push: ${{ inputs.push }} + tags: ${{ env.TAGS }} + cache-from: type=local,src=/tmp/.buildx-cache + cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + + # https://github.com/docker/build-push-action/issues/252 + # TODO: https://github.com/moby/buildkit/issues/1896 + - name: Move buildx cache + run: | + rm -rf /tmp/.buildx-cache + mv /tmp/.buildx-cache-new /tmp/.buildx-cache diff --git a/.github/actions/build-server/action.yml b/.github/actions/build-server/action.yml index d8d944b43..f8a96739a 100644 --- a/.github/actions/build-server/action.yml +++ b/.github/actions/build-server/action.yml @@ -1,9 +1,9 @@ -name: 'Compile rust' -description: 'Compile stump rust server' +name: 'Build Stump server' +description: 'Compile the Stump Rust server' inputs: platform: - description: 'runner plaform' + description: 'The plaform of the runner' required: true runs: @@ -12,21 +12,26 @@ runs: - name: Checkout project uses: actions/checkout@v3 + # - name: Configure environment + # run: | + # if [[ ${{ inputs.platform }} == 'linux' || ${{ inputs.platform }} == 'windows' ]]; then + # echo "RUN_SETUP=false" >> $GITHUB_ENV + # else + # echo "RUN_SETUP=true" >> $GITHUB_ENV + # fi + - name: Setup rust - uses: ./.github/actions/setup-system + uses: ./.github/actions/setup-cargo - name: Generate Prisma client uses: ./.github/actions/setup-prisma - - name: Download frontend + + - name: Copy bundled web app uses: actions/download-artifact@v3 with: - name: web + name: webapp path: ./apps/server/dist - - name: Copy web app to tauri - run: cp -r ./apps/server/dist apps/desktop/dist - shell: bash - - - name: Build server + - name: Compile server shell: bash - run: cargo build --release + run: cargo build --package stump_server --release diff --git a/.github/actions/build-web/action.yml b/.github/actions/build-web/action.yml index a72795a25..9a39f4517 100644 --- a/.github/actions/build-web/action.yml +++ b/.github/actions/build-web/action.yml @@ -1,14 +1,27 @@ -name: 'Build Web Application' -description: "Build web application and upload it's artifacts" +name: 'Compile Web Application' +description: 'Compile stump web' runs: - using: 'composite' + using: composite steps: - - name: Build - uses: ./.github/actions/compile-web + - name: Checkout project + uses: actions/checkout@v3 - - name: Upload + - name: Setup pnpm + uses: ./.github/actions/setup-pnpm + + - name: Install dependencies + shell: bash + run: pnpm install + working-directory: apps/web + + - name: Build app + shell: bash + run: pnpm run build + working-directory: apps/web + + - name: Upload bundle uses: ./.github/actions/upload-artifact with: - upload-name: web + upload-name: webapp upload-path: apps/web/dist diff --git a/.github/actions/compile-web/action.yml b/.github/actions/compile-web/action.yml deleted file mode 100644 index efcf33345..000000000 --- a/.github/actions/compile-web/action.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: 'Compile Web Application' -description: 'Compile stump web' - -runs: - using: composite - steps: - - name: Checkout project - uses: actions/checkout@v3 - - - uses: ./.github/actions/setup-pnpm - - - name: Install dependencies - shell: bash - run: pnpm install - working-directory: apps/web - - - name: Build app - shell: bash - run: pnpm run build - working-directory: apps/web diff --git a/.github/actions/setup-cargo/action.yml b/.github/actions/setup-cargo/action.yml index ebcb1300d..0c2ac9f73 100644 --- a/.github/actions/setup-cargo/action.yml +++ b/.github/actions/setup-cargo/action.yml @@ -4,13 +4,16 @@ description: 'Install system dependencies and setup cache' runs: using: 'composite' steps: + - name: Configure environment + run: | + if [[ ${{ runner.name }} == 'manjaro-az' || ${{ runner.os }} == 'Windows' ]]; then + echo "RUN_SETUP=false" >> $GITHUB_ENV + else + echo "RUN_SETUP=true" >> $GITHUB_ENV + fi + - name: System setup - # TODO: determine if this is actually OK... I keep getting locked out of sudo - # because the setup script has a few sudo commands that fail. I either need to setup - # some sort of askpass or (ideally) not run setup if it isn't needed on the self-hosted runner... - # Only run if we're NOT on a self-hosted runner. This check is really naive and should - # be changed. I doubt I'll host more than one runner, so this is fine for now. - if: runner.name != 'manjaro-az' + if: ${{ env.RUN_SETUP == 'true' }} shell: bash run: CHECK_NODE=0 CHECK_CARGO=0 DEV_SETUP=0 ./scripts/system-setup.sh diff --git a/.github/actions/upload-artifact/action.yml b/.github/actions/upload-artifact/action.yml index 1073af517..c7a0beb51 100644 --- a/.github/actions/upload-artifact/action.yml +++ b/.github/actions/upload-artifact/action.yml @@ -2,7 +2,6 @@ name: 'Upload Local' description: 'Upload artifact to local action' inputs: - # Upload upload-name: required: true description: 'Name of the upload' diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index df425e27e..4e500bf24 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -6,6 +6,7 @@ on: branches: - main +# TODO: figure out how to use moon here. jobs: check-rust: name: Rust checks @@ -36,10 +37,10 @@ jobs: - name: Checkout repository uses: actions/checkout@v3 - - name: Setup pnpm and typescript + - name: Setup PNPM and TypeScript uses: ./.github/actions/setup-pnpm - - name: lint + - name: Run TypeScript lints run: pnpm lint # - name: typecheck diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4483d88bf..5bdaca090 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -4,136 +4,160 @@ on: pull_request: branches: - develop + - main push: branches: - develop +# TODO: should I push nightly on main pushes? then on tag, an actual tagged release? + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + jobs: - # TODO: extract this into a reusable action that takes tags as input docker-build: name: Build docker image runs-on: [self-hosted] steps: - - name: Checkout repository - uses: actions/checkout@v3 + # On PRs, we will only load the image into docker for the quickest platform + # (i.e. linux/amd64). This is mostly a smoke test, just rather ignorant verification + # that the image can be built. On pushes, we will actually build and push for + # all supported platforms. + - name: Configure environment + run: | + echo "LOAD=${{ github.event_name == 'pull_request' }}" >> $GITHUB_ENV + echo "PUSH=${{ github.event_name == 'push' }}" >> $GITHUB_ENV - - name: Get commit short sha - # run: echo "::set-output name=GIT_REV::$(git rev-parse --short "$GITHUB_SHA")" - run: echo "GIT_REV=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV + if [[ ${{ github.event_name }} == 'pull_request' ]]; then + echo "PLATFORMS=${{ vars.SUPPORTED_PR_DOCKER_PLATFORMS }}" >> $GITHUB_ENV + else + echo "PLATFORMS=${{ vars.SUPPORTED_DOCKER_PLATFORMS }}" >> $GITHUB_ENV + fi - - name: Setup rust - uses: ./.github/actions/setup-cargo + - name: Setup and build docker image + uses: ./.github/actions/build-docker + with: + tags: 'nightly' + load: ${{ env.LOAD }} + push: ${{ env.PUSH }} + platforms: ${{ env.PLATFORMS }} - - name: Generate Prisma client - uses: ./.github/actions/setup-prisma + # TODO: build executables for apple(x86_64,darwin?),linux(x86_64,arm64?), and windows(x86_64) + # These should be uploaded to the nightly release as artifacts. Old artifacts should be deleted + # before uploading new ones. - - name: Setup buildx platforms - # if github.event_name == 'pull_request', platform is only linux/amd64. Otherwise, we - # build for the existing vars.SUPPORTED_DOCKER_PLATFORMS - run: | - if [ "${{ github.event_name }}" == "pull_request" ]; then - echo "SUPPORTED_DOCKER_PLATFORMS=linux/amd64" >> $GITHUB_ENV - echo "SETUP_QEMU=0" >> $GITHUB_ENV - else - echo "SUPPORTED_DOCKER_PLATFORMS=${{ vars.SUPPORTED_DOCKER_PLATFORMS }}" >> $GITHUB_ENV - echo "SETUP_QEMU=1" >> $GITHUB_ENV - fi + build-web: + name: Bundle web app + runs-on: [self-hosted] + if: false # TODO: don't do that + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Build web + uses: ./.github/actions/build-web - - name: Setup Docker layers cache - uses: actions/cache@v3 + - name: Upload web build + uses: ./.github/actions/upload-artifact with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + upload-name: webapp + upload-path: apps/web/dist - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - # Only setup qemu if we're not only building for linux/amd64 - if: ${{ env.SETUP_QEMU == 1 }} + build-linux-server: + name: Compile server app (self-hosted linux) + needs: build-web + runs-on: [self-hosted] + if: false # TODO: don't do that + steps: + - name: Checkout repository + uses: actions/checkout@v3 - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + - name: Build server + uses: ./.github/actions/build-server + with: + platform: 'linux' - - name: Login to Docker Hub - uses: docker/login-action@v2 + - name: Upload stump server + uses: ./.github/actions/upload-artifact with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + upload-name: stump_server-linux + upload-path: target/release/stump_server + + build-server: + strategy: + fail-fast: true + matrix: + platform: [macos, windows] + name: Compile server app + needs: build-web + runs-on: ${{ matrix.platform }} + if: false # TODO: don't do that + steps: + - name: Checkout repository + uses: actions/checkout@v3 - - name: Run buildx build - uses: docker/build-push-action@v4 + - name: Build server + uses: ./.github/actions/build-server with: - context: . - build-args: | - "GIT_REV=${{ env.GIT_REV }}" - file: scripts/release/Dockerfile - platforms: ${{ env.SUPPORTED_DOCKER_PLATFORMS }} - # When a PR is opened against develop, we want to build the image but not push it. - load: ${{ github.event_name == 'pull_request' }} - # When a PR is merged into develop, we want to build and push the image. Note that - # unless there was a push without a PR, much of the docker build will come out of - # the cache. - push: ${{ github.event_name == 'push' }} - tags: aaronleopold/stump:nightly - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max - - # https://github.com/docker/build-push-action/issues/252 - # TODO: https://github.com/moby/buildkit/issues/1896 - - name: Move buildx cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - - # TODO: uncomment once I resolve cross compilation with my single self-hosted runner - # build-web: - # strategy: - # fail-fast: true - # matrix: - # plaform: [ubuntu-20.04] - # name: Build web application - # runs-on: ${{ matrix.plaform }} - # steps: - # - name: Checkout repository - # uses: actions/checkout@v3 + platform: ${{ matrix.platform }} - # - name: Build web - # uses: ./.github/actions/build-web + - name: Upload stump server + uses: ./.github/actions/upload-artifact + with: + upload-name: stump_server-${{ matrix.platform }} + upload-path: target/release/stump_server - # - name: Upload web build - # uses: ./.github/actions/upload-artifact - # with: - # upload-name: web - # upload-path: apps/web/dist - - # build-server: - # strategy: - # fail-fast: true - # matrix: - # plaform: [ubuntu-20.04] # We should probably build for window and macos - # name: Build server and desktop - # runs-on: ${{ matrix.plaform }} + # build-linux-desktop: + # name: Compile desktop app (self-hosted linux) # needs: build-web + # runs-on: [self-hosted] + # if: false # TODO: don't do that # steps: # - name: Checkout repository # uses: actions/checkout@v3 - # - name: Build server - # uses: ./.github/actions/build-server + # - name: Build desktop + # uses: ./.github/actions/build-desktop # with: - # platform: ${{ matrix.plaform }} - - # - name: Upload stump server - # uses: ./.github/actions/upload-artifact - # with: - # upload-name: stump_server - # upload-path: target/release/stump_server - - # - uses: tauri-apps/tauri-action@v0 + # platform: 'linux' # - name: Upload desktop # uses: ./.github/actions/upload-artifact # with: - # upload-name: stump-desktop + # upload-name: stump-desktop-linux # upload-path: target/release/bundle + + build-desktop: + strategy: + fail-fast: true + matrix: + platform: [macos, windows] + name: Compile desktop app + needs: build-web + runs-on: ${{ matrix.platform }} + if: false # TODO: don't do that + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Build desktop + uses: ./.github/actions/build-desktop + with: + platform: ${{ matrix.platform }} + + # https://github.com/tauri-apps/tauri-action + # - uses: tauri-apps/tauri-action@v0 + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # tagName: stump-desktop-v__VERSION__ # the action automatically replaces \_\_VERSION\_\_ with the app version + # releaseName: 'Stump Desktop v__VERSION__' + # releaseBody: 'See the assets to download this version and install.' + # releaseDraft: true + # prerelease: true + + # - name: Upload desktop + # uses: ./.github/actions/upload-artifact + # with: + # upload-name: stump-desktop-${{ matrix.platform }} + # upload-path: target/release/bundle diff --git a/.vscode/.todo b/.vscode/.todo index 19c4b60d6..ea96ecb86 100644 --- a/.vscode/.todo +++ b/.vscode/.todo @@ -76,4 +76,5 @@ - [ ] Make light mode look not disgusting lmao - [ ] ~Fix poor performance of EditLibraryModal~ - [ ] Just replace the modals with dedicated pages for editing and creating libraries -- [ ] `local-ip-address` doesn't compile to arm64.... need to either guard it or correct it. +- [ ] `local-ip-address` check if release yoinked comes back okay +- [ ] cargo report future-incompatibilities --id 4 --package rustc-serialize@0.3.24 diff --git a/Cargo.toml b/Cargo.toml index ac2f92b77..37ed18db0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ members = [ [workspace.package] version = "0.0.0" -rust-version = "1.64.0" +rust-version = "1.68.0" [workspace.dependencies] prisma-client-rust = { git = "https://github.com/Brendonovich/prisma-client-rust.git", tag = "0.6.4", features = [ diff --git a/core/src/db/models/log.rs b/core/src/db/models/log.rs index 1ce04b2f3..0059a0a84 100644 --- a/core/src/db/models/log.rs +++ b/core/src/db/models/log.rs @@ -15,24 +15,19 @@ pub struct LogMetadata { pub modified: String, } -#[derive(Clone, Serialize, Deserialize, Type, ToSchema)] +#[derive(Clone, Default, Serialize, Deserialize, Type, ToSchema)] pub enum LogLevel { #[serde(rename = "ERROR")] Error, #[serde(rename = "WARN")] Warn, #[serde(rename = "INFO")] + #[default] Info, #[serde(rename = "DEBUG")] Debug, } -impl Default for LogLevel { - fn default() -> Self { - LogLevel::Info - } -} - impl std::fmt::Display for LogLevel { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { diff --git a/core/src/prelude/enums.rs b/core/src/prelude/enums.rs index ed25d0422..f081efc6a 100644 --- a/core/src/prelude/enums.rs +++ b/core/src/prelude/enums.rs @@ -4,11 +4,12 @@ use serde::{Deserialize, Serialize}; use specta::Type; use utoipa::ToSchema; -#[derive(Serialize, Deserialize, Type, ToSchema)] +#[derive(Serialize, Deserialize, Type, ToSchema, Default)] pub enum UserRole { #[serde(rename = "SERVER_OWNER")] ServerOwner, #[serde(rename = "MEMBER")] + #[default] Member, } @@ -67,12 +68,6 @@ impl FromStr for FileStatus { } } -impl Default for UserRole { - fn default() -> Self { - UserRole::Member - } -} - impl From for String { fn from(role: UserRole) -> String { match role { diff --git a/core/src/prelude/server/query.rs b/core/src/prelude/server/query.rs index fcebd3905..1358bafe4 100644 --- a/core/src/prelude/server/query.rs +++ b/core/src/prelude/server/query.rs @@ -7,20 +7,15 @@ use crate::{ prisma::{library, media, series}, }; -#[derive(Debug, Serialize, Deserialize, Clone, Type, ToSchema)] +#[derive(Debug, Default, Serialize, Deserialize, Clone, Type, ToSchema)] pub enum Direction { #[serde(rename = "asc")] Asc, #[serde(rename = "desc")] + #[default] Desc, } -impl Default for Direction { - fn default() -> Self { - Direction::Asc - } -} - impl From for prisma_client_rust::Direction { fn from(direction: Direction) -> prisma_client_rust::Direction { match direction { diff --git a/scripts/lib b/scripts/lib index e3c8ae23c..203e9b69e 100644 --- a/scripts/lib +++ b/scripts/lib @@ -1,5 +1,10 @@ #!/usr/bin/env bash +log_error() { + echo $1 1>&2 + exit 1 +} + create_dummy_rust_file() { local path=$1 echo "Creating dummy Rust file in $path" diff --git a/scripts/release/compile-server.sh b/scripts/release/compile-server.sh new file mode 100755 index 000000000..628346972 --- /dev/null +++ b/scripts/release/compile-server.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +SCRIPTS_DIR="${BASH_SOURCE%/*}/.." +source "${SCRIPTS_DIR}/lib" + +echo "Ensuring targets for apple(x86_64,darwin),linux(x86_64,arm64), and windows(x86_64) are installed..." + +set -ex; \ + rustup target add x86_64-apple-darwin; \ + rustup target add aarch64-apple-darwin; \ + rustup target add x86_64-unknown-linux-gnu; \ + rustup target add aarch64-unknown-linux-gnu; \ + rustup target add x86_64-pc-windows-gnu; \ + set +x + +# https://www.shogan.co.uk/development/rust-cross-compile-linux-to-macos-using-github-actions/ +# https://stackoverflow.com/questions/66849112/how-do-i-cross-compile-a-rust-application-from-macos-x86-to-macos-silicon +# https://gist.github.com/shqld/256e2c4f4b97957fb0ec250cdc6dc463 +# lol, at this point, I think it might just be easier to use GH hosted runners +# for the executable builds... Which would mean instead of this scripting doing it all, +# it would just run a subset of these operations on a per-os basis. E.g. the linux +# build would just run the linux-specific commands, and the macos build would just +# run the macos-specific commands. This would also mean that the build process +# would be a lot more straightforward, and would be a lot easier to maintain? Maybe to start, +# I trim it even further. Don't support both arm+x86, just do vanilla builds for each + +echo "Targets installed." + +CALL_TO_ACTION_LOL="Please consider helping to expand support for your system: https://github.com/aaronleopold/stump/issues" + +if [[ "$OSTYPE" == "linux-gnu"* ]]; then + UNSUPPORTED_DISTRO="Your distro '$(lsb_release -s -d)' is not supported by this script. $CALL_TO_ACTION_LOL" + + if which apt-get &> /dev/null; then + # TODO: add support for other distros + log_error "$UNSUPPORTED_DISTRO" + elif which pacman &> /dev/null; then + set -ex; \ + sudo pacman -S --needed mingw-w64-gcc + elif which dnf &> /dev/null; then + # TODO: add support for other distros + log_error "$UNSUPPORTED_DISTRO" + else + log_error "$UNSUPPORTED_DISTRO" + fi +elif [[ "$OSTYPE" == "darwin"* ]]; then + set -ex; \ + set HOMEBREW_NO_AUTO_UPDATE=1; \ + brew tap messense/macos-cross-toolchains; \ + brew install filosottile/musl-cross/musl-cross mingw-w64 x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu; \ + set +x + + if which musl-gcc &> /dev/null; then + echo "musl-gcc installed successfully." + else + # https://github.com/FiloSottile/homebrew-musl-cross + echo "musl-gcc is not detected, attempting symlink workaround..." + DIR="/usr/local/opt/musl-cross/bin/x86_64-linux-musl-gcc" + TARG="/usr/local/bin/musl-gcc" + if ln -s "$DIR" "$TARG"; then + echo "Symlink created successfully." + else + log_error "Symlink creation failed." + fi + fi +else + log_error "Your OS '$OSTYPE' is not supported by this script. $CALL_TO_ACTION_LOL" +fi + +export CC_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-gcc +export CXX_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-g++ +export AR_x86_64_unknown_linux_gnu=x86_64-unknown-linux-gnu-ar +export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=x86_64-unknown-linux-gnu-gcc + +cargo build --package stump_server --release \ + --target x86_64-apple-darwin \ + --target aarch64-apple-darwin \ + --target x86_64-unknown-linux-gnu \ + --target x86_64-unknown-linux-musl \ + --target x86_64-pc-windows-gnu diff --git a/scripts/system-setup.sh b/scripts/system-setup.sh index 5086d7d5e..81af7195e 100755 --- a/scripts/system-setup.sh +++ b/scripts/system-setup.sh @@ -1,22 +1,18 @@ #!/bin/bash +SCRIPTS_DIR="${BASH_SOURCE%/*}" +source "${SCRIPTS_DIR}/lib" + _DEV_SETUP=${DEV_SETUP:=1} _CHECK_CARGO=${CHECK_CARGO:=1} _CHECK_NODE=${CHECK_NODE:=1} _FORCE_INSTALL_PNPM=${INSTALL_PNPM:=0} dev_setup() { - echo "Installing 'cargo-watch'..." - echo - - cargo install cargo-watch - - echo "Running 'pnpm run setup'..." - echo - - pnpm run setup - - echo + set -ex; \ + cargo install cargo-watch; \ + pnpm run setup; \ + set +x } if [ "$name" == "nix-shell" ]; then @@ -27,8 +23,7 @@ fi if [ ${_CHECK_CARGO} == 1 ]; then which cargo &> /dev/null if [ $? -ne 0 ]; then - echo "Rust could not be found on your system. Visit https://www.rust-lang.org/tools/install" - exit 1 + log_error "Rust could not be found on your system. Visit https://www.rust-lang.org/tools/install" else echo "Rust requirement met!" fi @@ -37,8 +32,7 @@ fi if [ ${_CHECK_NODE} == 1 ]; then which node &> /dev/null if [ $? -eq 1 ]; then - echo "Node could not be found on your system. Visit https://nodejs.org/en/download/" - exit 1 + log_error "Node could not be found on your system. Visit https://nodejs.org/en/download/" else echo "Node requirement met!" fi @@ -63,9 +57,8 @@ if [ ${_CHECK_NODE} == 1 ]; then echo "pnpm installed successfully." can_continue=true else - echo "pnpm could not be installed. Please ensure you have node and npm installed." can_continue=false - exit 1 + log_error "pnpm could not be installed. Please ensure you have node and npm installed." fi ;; n) @@ -88,9 +81,13 @@ if [ ${_CHECK_NODE} == 1 ]; then fi fi -# TODO: group these? +CALL_TO_ACTION_LOL="Please consider helping to expand support for your system: https://github.com/aaronleopold/stump/issues" + +# TODO: group these? so lines aren't so long... # https://tauri.app/v1/guides/getting-started/prerequisites/#1-system-dependencies if [[ "$OSTYPE" == "linux-gnu"* ]]; then + UNSUPPORTED_DISTRO="Your distro '$(lsb_release -s -d)' is not supported by this script. $CALL_TO_ACTION_LOL" + if which apt-get &> /dev/null; then sudo apt-get -y update sudo apt-get -y install pkg-config libssl-dev libdbus-1-dev libsoup2.4-dev libwebkit2gtk-4.0-dev curl wget libgtk-3-dev libappindicator3-dev librsvg2-dev build-essential libayatana-appindicator3-dev @@ -102,8 +99,7 @@ if [[ "$OSTYPE" == "linux-gnu"* ]]; then sudo dnf install openssl-devel webkit2gtk4.0-devel curl wget libappindicator-gtk3 librsvg2-devel sudo dnf group install "C Development Tools and Libraries" else - echo "Your distro '$(lsb_release -s -d)' is not supported by this script. Please consider adding support for it: https://github.com/aaronleopold/stump/issues" - exit 1 + log_error $UNSUPPORTED_DISTRO fi if [ {$_DEV_SETUP} == 1 ]; then @@ -118,6 +114,5 @@ elif [[ "$OSTYPE" == "darwin"* ]]; then echo "Setup completed! Run 'pnpm dev:web' or 'pnpm start:web' to get started." else - echo "Your OS '$OSTYPE' is not supported by the pre-setup script. Please consider adding support for it: https://github.com/aaronleopold/stump/issues" - exit 1 + log_error "Your OS '$OSTYPE' is not supported by the pre-setup script. $CALL_TO_ACTION_LOL" fi From 6d4d9d3ed313b7855b0153f438e1bd90510e37eb Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:16:34 -0700 Subject: [PATCH 33/40] ugh --- .github/actions/build-docker/action.yml | 1 + .github/actions/setup-cargo/action.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml index e92a5d189..c64b82800 100644 --- a/.github/actions/build-docker/action.yml +++ b/.github/actions/build-docker/action.yml @@ -27,6 +27,7 @@ runs: - name: Format tags run: | echo "TAGS=$(echo ${{ inputs.tags }} | sed -e 's/,/,aaronleopold\/stump:/g' | sed -e 's/^/aaronleopold\/stump:/')" >> $GITHUB_ENV + shell: bash - name: Setup rust uses: ./.github/actions/setup-cargo diff --git a/.github/actions/setup-cargo/action.yml b/.github/actions/setup-cargo/action.yml index 0c2ac9f73..84bf59567 100644 --- a/.github/actions/setup-cargo/action.yml +++ b/.github/actions/setup-cargo/action.yml @@ -11,6 +11,7 @@ runs: else echo "RUN_SETUP=true" >> $GITHUB_ENV fi + shell: bash - name: System setup if: ${{ env.RUN_SETUP == 'true' }} From 7a50b4232ec471df2946b3bf8c85ceeb77364d07 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:23:27 -0700 Subject: [PATCH 34/40] ugh --- .github/actions/build-docker/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml index c64b82800..badc6628e 100644 --- a/.github/actions/build-docker/action.yml +++ b/.github/actions/build-docker/action.yml @@ -23,6 +23,7 @@ runs: - name: Get commit short sha run: echo "GIT_REV=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV + shell: bash - name: Format tags run: | @@ -52,6 +53,7 @@ runs: else echo "SETUP_QEMU=0" >> $GITHUB_OUTPUT fi + shell: bash - name: Set up QEMU uses: docker/setup-qemu-action@v2 @@ -86,3 +88,4 @@ runs: run: | rm -rf /tmp/.buildx-cache mv /tmp/.buildx-cache-new /tmp/.buildx-cache + shell: bash From bca9ab139e521d402b5ea6620d3400aefd29b7bf Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:24:06 -0700 Subject: [PATCH 35/40] just for now --- .github/workflows/ci.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4e500bf24..01a97dba3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,6 +9,7 @@ on: # TODO: figure out how to use moon here. jobs: check-rust: + if: false name: Rust checks runs-on: [self-hosted] steps: @@ -31,6 +32,7 @@ jobs: # cargo integration-tests check-typescript: + if: false name: TypeScript checks runs-on: [self-hosted] steps: From 7c14f2c166cc7598dbbefb7163b947cba9aa1161 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:27:07 -0700 Subject: [PATCH 36/40] sigh --- .github/actions/build-docker/action.yml | 10 ++++++++-- .github/workflows/nightly.yml | 2 ++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml index badc6628e..0e604ce2a 100644 --- a/.github/actions/build-docker/action.yml +++ b/.github/actions/build-docker/action.yml @@ -2,6 +2,12 @@ name: 'Build docker image' description: 'Build and load or push a tagged docker image for stump' inputs: + username: + description: 'Username for docker login' + required: true + password: + description: 'Token for docker login' + required: true load: description: 'Set output-type to docker' default: true @@ -65,8 +71,8 @@ runs: - name: Login to Docker Hub uses: docker/login-action@v2 with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ inputs.username }} + password: ${{ inputs.password }} - name: Run buildx build uses: docker/build-push-action@v4 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5bdaca090..d6cf931fd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -38,6 +38,8 @@ jobs: - name: Setup and build docker image uses: ./.github/actions/build-docker with: + username: ${{ secrets.DOCKERHUB_USERNAME }} + password: ${{ secrets.DOCKERHUB_TOKEN }} tags: 'nightly' load: ${{ env.LOAD }} push: ${{ env.PUSH }} From 66c4e467f01fcc30e48a020ecd0287776db4aba2 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:32:43 -0700 Subject: [PATCH 37/40] HUH? --- .github/workflows/nightly.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index d6cf931fd..11214931b 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -15,6 +15,10 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true +env: + DOCKER_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKER_PASSWORD: ${{ secrets.DOCKERHUB_TOKEN }} + jobs: docker-build: name: Build docker image @@ -38,8 +42,8 @@ jobs: - name: Setup and build docker image uses: ./.github/actions/build-docker with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} + username: ${{ env.DOCKER_USERNAME }} + password: ${{ env.DOCKER_PASSWORD }} tags: 'nightly' load: ${{ env.LOAD }} push: ${{ env.PUSH }} From 3c5715d7fcbe6659d0548421a3fa55264c3e3fee Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:36:08 -0700 Subject: [PATCH 38/40] maybe? --- .github/actions/build-docker/action.yml | 3 --- .github/workflows/nightly.yml | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml index 0e604ce2a..d612c849b 100644 --- a/.github/actions/build-docker/action.yml +++ b/.github/actions/build-docker/action.yml @@ -24,9 +24,6 @@ inputs: runs: using: composite steps: - - name: Checkout repository - uses: actions/checkout@v3 - - name: Get commit short sha run: echo "GIT_REV=$(git rev-parse --short "$GITHUB_SHA")" >> $GITHUB_ENV shell: bash diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 11214931b..68cdbd948 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -24,6 +24,9 @@ jobs: name: Build docker image runs-on: [self-hosted] steps: + - name: Checkout repository + uses: actions/checkout@v3 + # On PRs, we will only load the image into docker for the quickest platform # (i.e. linux/amd64). This is mostly a smoke test, just rather ignorant verification # that the image can be built. On pushes, we will actually build and push for From da9fd531324b5b52552bd6a336a28fb938e98b41 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:43:19 -0700 Subject: [PATCH 39/40] wrapping this up --- .github/workflows/build_nix.yml | 2 +- .github/workflows/ci.yaml | 4 +--- .github/workflows/nightly.yml | 2 +- README.md | 8 ++++---- package.json | 4 ---- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/build_nix.yml b/.github/workflows/build_nix.yml index 77abb1e56..af420f537 100644 --- a/.github/workflows/build_nix.yml +++ b/.github/workflows/build_nix.yml @@ -1,4 +1,4 @@ -name: 'Build legacy Nix package on Ubuntu' +name: 'Stump Nix CI' on: push: diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 01a97dba3..24a022a70 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,4 +1,4 @@ -name: Stump-CI +name: 'Stump Checks CI' on: pull_request: @@ -9,7 +9,6 @@ on: # TODO: figure out how to use moon here. jobs: check-rust: - if: false name: Rust checks runs-on: [self-hosted] steps: @@ -32,7 +31,6 @@ jobs: # cargo integration-tests check-typescript: - if: false name: TypeScript checks runs-on: [self-hosted] steps: diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 68cdbd948..e49c7655c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,4 +1,4 @@ -name: 'Stump nightly build' +name: 'Stump Nightly CI' on: pull_request: diff --git a/README.md b/README.md index f1b88fa2e..d5b3286a8 100644 --- a/README.md +++ b/README.md @@ -16,8 +16,8 @@ - - Docker Pulls + + Docker Pulls

@@ -37,7 +37,7 @@ Stump is a free and open source comics, manga and digital book server with OPDS - [Roadmap 🗺](#roadmap-) - [Getting Started 🚀](#getting-started-) - [Developer Guide 💻](#developer-guide-) - - [Where to start?](#where-to-start) + - [Where to start?](#where-to-start) - [Project Structure 📦](#project-structure-) - [/apps](#apps) - [/packages](#packages) @@ -80,7 +80,7 @@ I am very open to suggestions and ideas, so feel free to reach out if you have a Stump isn't ready for normal, non-development usage yet. Once a release has been made, this will be updated. For now, follow the [Developing](#developing-) section to build from source and run locally. -There is a [docker image](https://hub.docker.com/repository/docker/aaronleopold/stump-preview) available for those interested. However, **this is only meant for testing purposes and will not be updated frequently**, so do not expect a fully featured, bug-free experience if you spin up a container. Also keep in mind migrations won't be stacked until a release, so each update until then might require a wipe of the database file. +There is a [docker image](https://hub.docker.com/repository/docker/aaronleopold/stump) available for those interested. However, **this is only meant for testing purposes and will not be updated frequently**, so do not expect a fully featured, bug-free experience if you spin up a container. Also keep in mind migrations won't be stacked until a release, so each update until then might require a wipe of the database file. For more information about getting started, how Stump works and how it manages your library, and much more, please visit [stumpapp.dev](https://stumpapp.dev/guides). diff --git a/package.json b/package.json index e4378bd4f..0afa3348e 100644 --- a/package.json +++ b/package.json @@ -20,10 +20,6 @@ "build:server": "pnpm run server build", "build:web": "pnpm web build && pnpm build:server", "build:desktop": "pnpm desktop build", - "build:docker": "docker buildx build -f scripts/release/Dockerfile --push --platform=linux/arm64/v8,linux/amd64 -t aaronleopold/stump-preview:latest .", - "cache:docker": "docker buildx build -f scripts/release/Dockerfile --load --platform=linux/arm64/v8,linux/amd64 -t aaronleopold/stump-preview:latest .", - "cache:docker-arm": "docker buildx build -f scripts/release/Dockerfile --load --platform=linux/arm64/v8 -t aaronleopold/stump-preview:latest .", - "build:docker-amd": "docker buildx build -f scripts/release/Dockerfile --push --platform=linux/amd64 -t aaronleopold/stump-preview:latest .", "moon": "moon --color --log trace" }, "devDependencies": { From 58dd3c0fa39c0f442034618a7006ed077d697d54 Mon Sep 17 00:00:00 2001 From: Aaron Leopold <36278431+aaronleopold@users.noreply.github.com> Date: Fri, 10 Mar 2023 16:50:42 -0700 Subject: [PATCH 40/40] wrapping this up x2 --- .github/actions/build-docker/action.yml | 31 ++++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/.github/actions/build-docker/action.yml b/.github/actions/build-docker/action.yml index d612c849b..a0728c16f 100644 --- a/.github/actions/build-docker/action.yml +++ b/.github/actions/build-docker/action.yml @@ -39,13 +39,14 @@ runs: - name: Generate Prisma client uses: ./.github/actions/setup-prisma - - name: Setup Docker layers cache - uses: actions/cache@v3 - with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-buildx- + # TODO: uncomment once cache stuff is resolved... + # - name: Setup Docker layers cache + # uses: actions/cache@v3 + # with: + # path: /tmp/.buildx-cache + # key: ${{ runner.os }}-buildx-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-buildx- # We only need QEMU when an arm* platform is targeted - name: Check QEMU requirement @@ -82,13 +83,15 @@ runs: load: ${{ inputs.load }} push: ${{ inputs.push }} tags: ${{ env.TAGS }} - cache-from: type=local,src=/tmp/.buildx-cache - cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max + # TODO: uncomment once cache stuff is resolved... + # cache-from: type=local,src=/tmp/.buildx-cache + # cache-to: type=local,dest=/tmp/.buildx-cache-new,mode=max # https://github.com/docker/build-push-action/issues/252 # TODO: https://github.com/moby/buildkit/issues/1896 - - name: Move buildx cache - run: | - rm -rf /tmp/.buildx-cache - mv /tmp/.buildx-cache-new /tmp/.buildx-cache - shell: bash + # TODO: uncomment once cache stuff is resolved... + # - name: Move buildx cache + # run: | + # rm -rf /tmp/.buildx-cache + # mv /tmp/.buildx-cache-new /tmp/.buildx-cache + # shell: bash