init: working implementation

This is the first working implementation that I tested manually.

Formal tests to follow.

.github/scripts/cross.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -e
+
+if test "$BUILD_CMD" != "cross"
+then
+    echo "cross.sh - is a helper to assist only in cross compiling environments" >&2
+    echo "To use this tool set the BUILD_CMD env var to the \"cross\" value" >&2
+    exit 111
+fi
+
+if test -z "$CROSS_IMAGE"
+then
+    echo "The CROSS_IMAGE env var should be provided" >&2
+    exit 111
+fi
+
+docker run --rm -v /home/runner:/home/runner -w "$PWD" "$CROSS_IMAGE" "$@"

.github/workflows/build.yml

@@ -0,0 +1,102 @@
+name: Build & Test
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTFLAGS: "-D warnings"
+
+on:
+  workflow_call:
+    inputs:
+      run_test:
+        default: true
+        type: boolean
+
+jobs:
+  build:
+    name: ${{ matrix.platform }} (${{ matrix.target }}) (${{ matrix.os }})
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 40
+    strategy:
+      fail-fast: false
+      matrix:
+        platform:
+          - linux-arm
+          - linux-arm64
+          - linux-x64
+          - linux-x86
+          - macos-arm64
+          - macos-x64
+
+        include:
+          # When adding a new `target`:
+          # 1. Define a new platform alias above
+          # 2. Add a new record to a matrix map in `cli/npm/install.js`
+          - { platform: linux-arm         , target: arm-unknown-linux-gnueabi           , os: ubuntu-latest  }
+          - { platform: linux-arm64       , target: aarch64-unknown-linux-gnu           , os: ubuntu-latest  }
+          - { platform: linux-x64         , target: x86_64-unknown-linux-gnu            , os: ubuntu-20.04   }
+          - { platform: linux-x86         , target: i686-unknown-linux-gnu              , os: ubuntu-latest  }
+          - { platform: macos-arm64       , target: aarch64-apple-darwin                , os: macos-14       }
+          - { platform: macos-x64         , target: x86_64-apple-darwin                 , os: macos-12       }
+
+    env:
+      BUILD_CMD: cargo
+
+    defaults:
+      run:
+        shell: bash
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: rustup toolchain install stable --profile minimal
+      - run: rustup target add ${{ matrix.target }}
+
+      - name: Install cross
+        if: ${{ matrix.use-cross }}
+        uses: taiki-e/install-action@v2
+        with:
+          tool: cross
+
+      - name: Build custom cross image
+        if: ${{ matrix.use-cross && matrix.os == 'ubuntu-latest' }}
+        run: |
+          target="${{ matrix.target }}"
+          image=ghcr.io/cross-rs/$target:custom
+          echo "CROSS_IMAGE=$image"                              >> $GITHUB_ENV
+
+          echo "[target.$target]"                                >> Cross.toml
+          echo "image = \"$image\""                              >> Cross.toml
+          echo "CROSS_CONFIG=$PWD/Cross.toml"                    >> $GITHUB_ENV
+
+          echo "FROM ghcr.io/cross-rs/$target:edge"                            >> Dockerfile
+          echo "RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash -" >> Dockerfile
+          echo "RUN apt-get update && apt-get -y install nodejs"               >> Dockerfile
+          docker build -t $image .
+
+      - name: Setup env extras
+        env:
+          TARGET: ${{ matrix.target }}
+        run: |
+          PATH="$PWD/.github/scripts:$PATH"
+          echo "$PWD/.github/scripts" >> $GITHUB_PATH
+
+          echo "TARGET=$TARGET" >> $GITHUB_ENV
+          echo "ROOT=$PWD" >> $GITHUB_ENV
+
+          echo "BUILD_CMD=cross" >> $GITHUB_ENV
+          runner=$(BUILD_CMD=cross cross.sh bash -c "env | sed -nr '/^CARGO_TARGET_.*_RUNNER=/s///p'")
+          [ -n "$runner" ] && echo "CROSS_RUNNER=$runner" >> $GITHUB_ENV
+
+      - run: $BUILD_CMD build --release --target=${{ matrix.target }}
+
+      - name: Run main tests
+        if: ${{ inputs.run_test }}
+        run: $BUILD_CMD test --target=${{ matrix.target }}
+
+      - name: Upload CLI artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: tsp.${{ matrix.platform }}
+          path: target/${{ matrix.target }}/release/tsp
+          if-no-files-found: error
+          retention-days: 7

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+on:
+  pull_request:
+  push:
+    branches:
+      - "master"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.event_name != 'push' }}
+
+jobs:
+  checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: sudo apt install just
+      - run: rustup toolchain install stable --profile minimal
+      - run: rustup toolchain install nightly --profile minimal
+      - run: rustup component add --toolchain nightly rustfmt
+      - uses: Swatinem/rust-cache@v2
+      - run: just lint
+
+  build:
+    uses: ./.github/workflows/build.yml

.gitignore

@@ -0,0 +1,12 @@
+# rust
+debug/
+target/
+Cargo.lock
+
+# These are backup files generated by rustfmt
+**/*.rs.bk
+
+# tsp
+parsers/
+tmp/
+parsers.toml

Cargo.toml

@@ -0,0 +1,88 @@
+[package]
+authors = ["Firas al-Khalil <firasalkhalil@gmail.com>"]
+build = "build.rs"
+description = "A builder of many tree-sitter parsers"
+edition = "2021"
+name = "tsp"
+version = "0.4.2"
+
+[lib]
+name = "tsp"
+path = "src/lib.rs"
+
+[[bin]]
+name = "tsp"
+path = "src/main.rs"
+
+[package.metadata.tsp]
+build-dir = "tmp"
+config = "parsers.toml"
+fresh = false
+from = "https://github.com/tree-sitter/tree-sitter-"
+out = "parsers"
+prefix = "libtree-sitter-"
+ref = "master"
+show-config = false
+sys = false
+
+[package.metadata.tree-sitter]
+repo = "https://github.com/tree-sitter/tree-sitter"
+version = "0.23.0"
+
+[dependencies]
+atty = "0.2"
+better-panic = "0.3.0"
+clap = { version = "4.5", features = ["derive", "cargo"] }
+clap-verbosity-flag = "2.2"
+console = "0.15"
+derive_more = { version = "1.0", features = [
+  "as_ref",
+  "deref",
+  "display",
+  "from",
+  "from_str",
+  "into",
+] }
+diff-struct = "0.5"
+enum_dispatch = "0.3"
+figment = { version = "0.10", features = ["toml", "env"] }
+human-panic = "2.0.1"
+ignore = "0.4"
+indicatif = "0.17"
+log = "0.4"
+miette = { version = "7.2.0", features = ["fancy"] }
+num_cpus = "1.16"
+serde = { version = "1.0", features = ["derive"] }
+thiserror = "1"
+tokio = { version = "1", features = [
+  "fs",
+  "process",
+  "rt-multi-thread",
+  "sync",
+  "time",
+] }
+toml = "0.8"
+tracing = "0.1"
+tracing-appender = "0.2"
+tracing-error = "0.2"
+tracing-log = "0.2"
+tracing-subscriber = "0.3"
+url = "2.5"
+
+[dev-dependencies]
+assert_cmd = "2.0"
+assert_fs = "1.1"
+indoc = "2"
+predicates = "3.1"
+pretty_assertions = "1.4"
+rstest = "0.22.0"
+
+[build-dependencies]
+cargo_metadata = "0.18"
+const-str = "0.5"
+indoc = "2"
+serde_json = "1.0"
+
+[lints.clippy]
+pedantic = { level = "warn", priority = -1 }
+missing-errors-doc = "allow"

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2024 Firas al-Khalil
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,5 @@
+# TSP
+
+A genrator of tree-sitter parsers as dynamic or static libraries
+
+> :warning: Under Construction!

TODO.md

@@ -0,0 +1,51 @@
+# TODO
+
+## CI
+
+- [ ] Cross-platform builds: support all tree-sitter platforms.
+  - [ ] Package
+    - [ ] Linux
+    - [ ] Mac
+    - [ ] Tar / Zip
+  - [ ] Release binaries and separate linux/mac distribution packages.
+    - [ ] Github
+    - [ ] crates.io
+
+## Commands
+
+- [ ] check command: check that all tools necessary are installed (gunzip, wget, curl, git)
+
+## Configurtion
+
+- [ ] Investigate a figment replacement / custom impl to merge diferent configuration
+    sources.
+
+## Maintenance
+
+- [ ] A sane way to produce change logs
+- [ ] just release {{arg}}
+  - [ ] {{arg}} is a version number => tag with v{{args}}
+  - [ ] Handle changelog
+  - [ ] push to main repo with tags
+  - [ ] CI should kick in
+
+### Options
+
+- [ ] --sys-ts, false by default
+  - [x] Add the flag.
+  - [ ] Use [TREE_SITTER_LIBDIR](https://github.com/tree-sitter/tree-sitter/blob/4f97cf850535a7b23e648aba6e355caed1f10231/cli/loader/src/lib.rs#L177)
+        by default
+  - [ ] Use pkgconfig for sys libs
+
+## Tests
+
+- [ ] Use assert_cmd
+  - [ ] Test --force
+  - [ ] Test --sys-ts
+- [ ] Config
+  - [ ] with default config
+    - [ ] You can always download a parser even if it's not in the config.
+    - [ ] Verify it's actually HEAD when the parser is not in the config using git in the test.
+  - [ ] with custom config file.
+    - [ ] ask for parsers defined in the config file
+    - [ ] ask for parsers !defined in the config file and verify they're from the repo's HEAD.