init: working implementation

This is the first working implementation that I tested manually.

Formal tests to follow.

.github/scripts/cross.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+if test "$BUILD_CMD" != "cross"
+then
+    echo "cross.sh - is a helper to assist only in cross compiling environments" >&2
+    echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2
+    exit 111
+fi
+
+if test -z "$CROSS_IMAGE"
+then
+    echo "The CROSS_IMAGE env var should be provided" >&2
+    exit 111
+fi
+
+docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@"

.github/workflows/build.yml

@@ -0,0 +1,106 @@
+name: Build
+
+on:
+  workflow_call
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+jobs:
+  build:
+    name: ${{ matrix.platform }} (${{ matrix.target }}) (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux-arm
+          - linux-arm64
+          - linux-x64
+          - linux-x86
+          - macos-arm64
+          - macos-x64
+
+        include:
+          # When adding a new `target`:
+          # 1. Define a new platform alias above
+          # 2. Add a new record to a matrix map in `cli/npm/install.js`
+          - { platform: linux-arm   , target: arm-unknown-linux-gnueabi , os: ubuntu-latest  }
+          - { platform: linux-arm64 , target: aarch64-unknown-linux-gnu , os: ubuntu-latest  }
+          - { platform: linux-x64   , target: x86_64-unknown-linux-gnu  , os: ubuntu-latest  }
+          - { platform: linux-x86   , target: i686-unknown-linux-gnu    , os: ubuntu-latest  }
+          - { platform: macos-arm64 , target: aarch64-apple-darwin      , os: macos-14       }
+          - { platform: macos-x64   , target: x86_64-apple-darwin       , os: macos-12       }
+
+    env:
+      BUILD_CMD: cargo
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: rustup toolchain install stable --profile minimal
+      - run: rustup target add ${{ matrix.target }}
+
+      - name: Install cross
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cross
+
+      - name: Build custom cross image
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          target="${{ matrix.target }}"
+          image=ghcr.io/cross-rs/$target:custom
+          echo "CROSS_IMAGE=$image"                                              >> $GITHUB_ENV
+
+          echo "[target.$target]"                                                >> Cross.toml
+          echo "image = \"$image\""                                              >> Cross.toml
+          echo "CROSS_CONFIG=$PWD/Cross.toml"                                    >> $GITHUB_ENV
+
+          echo "FROM ghcr.io/cross-rs/$target:edge"                              >> Dockerfile
+          echo "RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash -"   >> Dockerfile
+          echo "RUN apt-get update && apt-get -y install nodejs"                 >> Dockerfile
+          docker build -t $image .
+
+      - name: Setup env extras
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        env:
+          TARGET: ${{ matrix.target }}
+        run: |
+          PATH="$PWD/.github/scripts:$PATH"
+          echo "$PWD/.github/scripts"                     >> $GITHUB_PATH
+
+          echo "ROOT=$PWD"                                >> $GITHUB_ENV
+          echo "TARGET=$TARGET"                           >> $GITHUB_ENV
+
+          echo "BUILD_CMD=cross"                          >> $GITHUB_ENV
+          runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
+          [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
+
+      - run: $BUILD_CMD build --release --target=${{ matrix.target }}
+
+      # It would have been great to test also the built binaries for each platform
+      # using cross, but I faced many issues, mainly on arm/aarch64:
+      # 1. assert_cmd should slip "--" between the command and its arguments
+      # 2. git, curl, chmod, etc … would work, but not tree-sitter-cli, for 2 reasons:
+      #    1. we need to recognize if we're being run with cross using CARGO_*_RUNNER envvar,
+      #       and if so, we need to modify the command + slip "--" after its args
+      #    2. but even if we did, tree-sitter-cli itself invokes node directly, and that
+      #       would not work because it doesn't recognize the runner.
+      #
+      # So I abandoned it. We have to rely on the fact that tree-sitter-cli is well tested
+      # on these platforms, and we need to make sure that tsdl itself runs well on linux/macOS.
+      - name: Upload CLI artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: tsdl.${{ matrix.platform }}
+          path: target/${{ matrix.target }}/release/tsdl
+          if-no-files-found: error
+          retention-days: 7

.github/workflows/ci.yml

@@ -0,0 +1,31 @@
+name: CI
+on:
+  pull_request:
+  push:
+    branches:
+      - "master"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: rustup toolchain install stable --profile minimal
+
+      - name: Install just
+        uses: taiki-e/install-action@v2
+        with:
+          tool: just
+
+      - run: just lint
+
+  build:
+    uses: ./.github/workflows/build.yml
+
+  test:
+    uses: ./.github/workflows/test.yml

.github/workflows/release.yml

@@ -0,0 +1,68 @@
+name: Release
+on:
+  workflow_dispatch:
+  push:
+    tags:
+      - v[0-9]+.[0-9]+.[0-9]+
+
+jobs:
+  build:
+    uses: ./.github/workflows/build.yml
+    with:
+      run_test: false
+
+  release:
+    name: Release
+    runs-on: ubuntu-latest
+    needs: build
+    permissions:
+      contents: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Download build artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+
+      - name: Display structure of downloaded files
+        run: ls -lR
+        working-directory: artifacts
+
+      - name: Prepare release artifacts
+        run: |
+          mkdir -p target
+          for platform in $(cd artifacts; ls | sed 's/^tsdl\.//'); do
+            exe=$(ls artifacts/tsdl.$platform/tsdl*)
+            gzip --stdout --name $exe > target/tsdl-$platform.gz
+          done
+          rm -rf artifacts
+          ls -l target/
+
+      - name: Create release
+        uses: softprops/action-gh-release@v2
+        with:
+          name: ${{ github.ref_name }}
+          tag_name: ${{ github.ref_name }}
+          fail_on_unmatched_files: true
+          files: |
+            target/tree-sitter-*.gz
+
+  crates_io:
+    name: Publish CLI to Crates.io
+    runs-on: ubuntu-latest
+    needs: release
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          profile: minimal
+          toolchain: stable
+          override: true
+
+      - name: Publish crates to Crates.io
+        uses: katyo/publish-crates@v2
+        with:
+          registry-token: ${{ secrets.CARGO_REGISTRY_TOKEN }}

.github/workflows/test.yml

@@ -0,0 +1,36 @@
+name: Test
+
+on:
+  workflow_call
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+jobs:
+  build:
+    name: ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        os:
+          - ubuntu-latest
+          - macos-latest
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: rustup toolchain install stable --profile minimal
+
+      - name: Install test tools
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cargo-nextest,just
+
+      - run: just test

.gitignore

@@ -0,0 +1,12 @@
+# rust
+debug/
+target/
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# tsdl
+parsers/
+tmp/
+parsers.toml

Cargo.toml

@@ -0,0 +1,88 @@
+[package]
+authors = ["Firas al-Khalil <firasalkhalil@gmail.com>"]
+build = "build.rs"
+description = "A downloader/builder of many tree-sitter parsers"
+edition = "2021"
+name = "tsdl"
+version = "0.99.0"
+
+[lib]
+name = "tsdl"
+path = "src/lib.rs"
+
+[[bin]]
+name = "tsdl"
+path = "src/main.rs"
+
+[package.metadata.tsdl]
+build-dir = "tmp"
+config = "parsers.toml"
+fresh = false
+from = "https://github.com/tree-sitter/tree-sitter-"
+out = "parsers"
+prefix = "libtree-sitter-"
+ref = "master"
+show-config = false
+sys = false
+
+[package.metadata.tree-sitter]
+repo = "https://github.com/tree-sitter/tree-sitter"
+version = "0.23.0"
+
+[dependencies]
+atty = "0.2"
+better-panic = "0.3.0"
+clap = { version = "4.5", features = ["derive", "cargo"] }
+clap-verbosity-flag = "2.2"
+console = "0.15"
+derive_more = { version = "1.0", features = [
+  "as_ref",
+  "deref",
+  "display",
+  "from",
+  "from_str",
+  "into",
+] }
+diff-struct = "0.5"
+enum_dispatch = "0.3"
+figment = { version = "0.10", features = ["toml", "env"] }
+human-panic = "2.0.1"
+ignore = "0.4"
+indicatif = "0.17"
+log = "0.4"
+miette = { version = "7.2.0", features = ["fancy"] }
+num_cpus = "1.16"
+serde = { version = "1.0", features = ["derive"] }
+thiserror = "1"
+tokio = { version = "1", features = [
+  "fs",
+  "process",
+  "rt-multi-thread",
+  "sync",
+  "time",
+] }
+toml = "0.8"
+tracing = "0.1"
+tracing-appender = "0.2"
+tracing-error = "0.2"
+tracing-log = "0.2"
+tracing-subscriber = "0.3"
+url = "2.5"
+
+[dev-dependencies]
+assert_cmd = "2.0"
+assert_fs = "1.1"
+indoc = "2"
+predicates = "3.1"
+pretty_assertions = "1.4"
+rstest = "0.22.0"
+
+[build-dependencies]
+cargo_metadata = "0.18"
+const-str = "0.5"
+indoc = "2"
+serde_json = "1.0"
+
+[lints.clippy]
+pedantic = { level = "warn", priority = -1 }
+missing-errors-doc = "allow"

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Firas al-Khalil
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,5 @@
+# TSDL
+
+A genrator of tree-sitter parsers as dynamic or static libraries
+
+> :warning: Under Construction!