From 3c5af115a7e3b9867265426d46dda72305f63e43 Mon Sep 17 00:00:00 2001 From: David Coles Date: Thu, 30 Mar 2023 16:32:51 -0700 Subject: [PATCH] Generate bindings at compile-time This replaces the `generate-bindings` script with a `build.rs` script for `flipperzero-sys` that downloads the latest SDK and uses that to generate Rust bindings. Fixes: #41 --- .github/workflows/update_bindings.yml | 88 -------- crates/sys/Cargo.toml | 11 + crates/sys/build.rs | 302 +++++++++++++++++++++++++ docs/updating-sdk.md | 47 +--- tools/Cargo.lock | 267 +--------------------- tools/Cargo.toml | 7 - tools/Dockerfile | 64 ------ tools/README.md | 4 - tools/src/bin/generate-bindings.rs | 304 -------------------------- 9 files changed, 322 insertions(+), 772 deletions(-) delete mode 100644 .github/workflows/update_bindings.yml create mode 100644 crates/sys/build.rs delete mode 100644 tools/Dockerfile delete mode 100644 tools/src/bin/generate-bindings.rs diff --git a/.github/workflows/update_bindings.yml b/.github/workflows/update_bindings.yml deleted file mode 100644 index eb881cd2..00000000 --- a/.github/workflows/update_bindings.yml +++ /dev/null @@ -1,88 +0,0 @@ -name: Update Bindings - -on: - workflow_dispatch: - inputs: - sdk_version: - description: flipperzero-firmware SDK version - type: string - required: true - -jobs: - sdk: - name: Update SDK bindings - runs-on: ubuntu-latest - permissions: - # SAFETY: A commit authored by the Bot will be pushed - contents: write - # SAFETY: An update PR will be created by the Bot - pull-requests: write - steps: - - - name: Validate SDK version - run: echo '${{ github.event.inputs.sdk_version }}' | grep --perl-regexp '^\d+\.\d+\.\d+$' - - - name: Checkout sources - uses: actions/checkout@eef61447b9ff4aafe5dcd4e0bbf5d482be7e7871 # v4.2.1 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@c47758b77c9736f4b2ef4073d4d51994fabfe349 # v3.7.1 - - - name: Create branch - run: git checkout -b'github-actions/update-bindings/sdk/${{ github.event.inputs.sdk_version }}' - - - name: Generate bindings - id: generate-bindings - uses: docker/build-push-action@4f58ea79222b3b9dc2c8bbdd6debcef730109a75 # v6.9.0 - with: - context: . - load: true - build-args: 'BRANCH=${{ github.event.inputs.sdk_version }}' - file: tools/Dockerfile - cache-from: type=gha - cache-to: type=gha,mode=max - - - name: Copy bindings - run: | - container="$(docker container create ${{ steps.generate-bindings.outputs.imageid }} --read-only)" - docker container cp "${container}":bindings.rs ./crates/sys/src/bindings.rs - docker container rm "${container}" - - - name: Commit changes - run: | - git config user.name 'github-actions[bot]' - git config user.email 'github-actions[bot]@users.noreply.github.com' - git commit crates/sys/src/bindings.rs \ - --message='build(bindings): bump SDK to `${{ github.event.inputs.sdk_version }}`' - - - name: Push changes - run: git push --set-upstream origin 'github-actions/update-bindings/sdk/${{ github.event.inputs.sdk_version }}' - - - name: Create update Pull Request - run: | - gh pr create \ - --repo='flipperzero-rs/flipperzero' \ - --base='main' \ - --title='build(bindings): bump SDK to `${{ github.event.inputs.sdk_version }}`' \ - --body="${PULL_REQUEST_DESCRIPTION}" \ - --label='sdk-update' - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PULL_REQUEST_DESCRIPTION: >- - # Description - - - This updates SDK bindings to [`${{ github.event.inputs.sdk_version }}`][1]. - - - --- - - - This PR has been automatically created by user @${{ github.triggering_actor }} - via `${{ github.workflow }}` workflow. - - - Further changes may added to this pull request. - - - [1]: https://github.com/flipperdevices/flipperzero-firmware/releases/tag/${{ github.event.inputs.sdk_version }} diff --git a/crates/sys/Cargo.toml b/crates/sys/Cargo.toml index 93a56f9e..de967513 100644 --- a/crates/sys/Cargo.toml +++ b/crates/sys/Cargo.toml @@ -23,3 +23,14 @@ test = false [dependencies] ufmt.workspace = true + +[build-dependencies] +bindgen = "0.63.0" +camino = "1.1.2" +curl = "0.4.44" +clap = { version = "4.0.32", features = ["cargo"] } +csv = "1.1.6" +serde = { version = "1.0.152", features = ["derive"] } +serde_json = "1.0.91" +shlex = "1.1.0" +zip = "0.6.4" diff --git a/crates/sys/build.rs b/crates/sys/build.rs new file mode 100644 index 00000000..c8f8cca2 --- /dev/null +++ b/crates/sys/build.rs @@ -0,0 +1,302 @@ +//! Build script for `flipperzero-sys` +//! +//! The targeted release and toolchain versions can be overridden with the following environment variables: +//! - `FLIPPER_SDK_VERSION` +//! - `FLIPPER_TOOLCHAIN_VERSION` + +use std::io::{self, Write}; +use std::path::PathBuf; +use std::{env, fs}; + +use camino::{Utf8Path, Utf8PathBuf}; +use curl::easy::Easy; +use serde::Deserialize; + +const FLIPPER_SDK_BASE: &str = "flipper-z-f7-sdk"; +const FLIPPER_SDK_VERSION: &str = "1.0.1"; +const FLIPPER_TOOLCHAIN_BASE: &str = "gcc-arm-none-eabi-12.3"; +const FLIPPER_TOOLCHAIN_VERSION: &str = "39"; + +const BUILDS_URL: &str = "https://update.flipperzero.one/builds"; + +#[cfg(target_arch = "x86_64")] +const FLIPPER_TOOLCHAIN_ARCH: &str = "x86_64"; +#[cfg(target_arch = "aarch64")] +const FLIPPER_TOOLCHAIN_ARCH: &str = "arm64"; + +#[cfg(target_os = "windows")] +const FLIPPER_TOOLCHAIN_OS: &str = "windows"; +#[cfg(target_os = "macos")] +const FLIPPER_TOOLCHAIN_OS: &str = "darwin"; +#[cfg(target_os = "linux")] +const FLIPPER_TOOLCHAIN_OS: &str = "linux"; + +const TARGET: &str = "thumbv7em-none-eabihf"; +const OUTFILE: &str = "bindings.rs"; +const SDK_OPTS: &str = "sdk.opts"; +const VISIBILITY_PUBLIC: &str = "+"; + +#[derive(Debug)] +struct ApiSymbols { + pub api_version: u32, + pub headers: Vec, + pub functions: Vec, + pub variables: Vec, +} + +/// Fetch Flipper Zero SDK. +fn fetch_sdk(download_dir: impl AsRef) -> io::Result { + let version = + env::var("FLIPPER_SDK_VERSION").unwrap_or_else(|_| FLIPPER_SDK_VERSION.to_string()); + let sdk = format!("{FLIPPER_SDK_BASE}-{version}"); + let filename = format!("{sdk}.zip"); + + let download_path = download_dir.as_ref().join(&filename); + + if !download_path.exists() { + // Fetch SDK + download( + &download_path, + &format!("{BUILDS_URL}/firmware/{FLIPPER_SDK_VERSION}/{filename}"), + )?; + } + + let sdk_dir = download_dir.as_ref().join(&sdk); + + if !sdk_dir.exists() { + // Extract SDK + let file = fs::File::open(&download_path)?; + let mut zip = zip::ZipArchive::new(file)?; + + zip.extract(&sdk_dir)?; + } + + Ok(sdk_dir) +} + +/// Fetch Flipper Zero toolchain. +fn fetch_toolchain(download_dir: impl AsRef) -> io::Result { + let toolchain = + format!("{FLIPPER_TOOLCHAIN_BASE}-{FLIPPER_TOOLCHAIN_ARCH}-{FLIPPER_TOOLCHAIN_OS}-flipper"); + let version = + env::var("FLIPPER_TOOLCHAIN_VERSION").unwrap_or_else(|_| FLIPPER_TOOLCHAIN_VERSION.into()); + let filename = format!("{toolchain}-{version}.zip"); + + let download_path = download_dir.as_ref().join(&filename); + + if !download_path.exists() { + // Fetch SDK + download( + &download_path, + &format!("{BUILDS_URL}/toolchain/{filename}"), + )?; + } + + let toolchain_dir = download_dir.as_ref().join(format!("{toolchain}-{version}")); + + if !toolchain_dir.exists() { + // Extract SDK + let file = fs::File::open(&download_path)?; + let mut zip = zip::ZipArchive::new(file)?; + + zip.extract(&toolchain_dir)?; + } + + Ok(toolchain_dir.join(toolchain)) +} + +/// Download contents of URL to a path. +fn download(path: impl AsRef, url: &str) -> io::Result<()> { + let mut f = fs::File::create(path.as_ref())?; + + let mut easy = Easy::new(); + easy.url(url)?; + easy.write_function(move |data| { + f.write_all(data).unwrap(); + + Ok(data.len()) + })?; + easy.perform()?; + + Ok(()) +} + +/// Load symbols from `api_symbols.csv`. +fn load_symbols>(path: T) -> ApiSymbols { + let path = path.as_ref(); + + let mut reader = csv::Reader::from_path(path).expect("failed to load symbol file"); + + let mut api_version: u32 = 0; + let mut headers = Vec::new(); + let mut functions = Vec::new(); + let mut variables = Vec::new(); + + for record in reader.records() { + let record = record.expect("failed to parse symbol record"); + let name = &record[0]; + let visibility = &record[1]; + let value = &record[2]; + + if visibility != VISIBILITY_PUBLIC { + continue; + } + + match name { + "Version" => { + let v = value + .split_once('.') + .expect("failed to parse symbol version"); + let major: u16 = v.0.parse().unwrap(); + let minor: u16 = v.1.parse().unwrap(); + + api_version = ((major as u32) << 16) | (minor as u32); + } + "Header" => headers.push(value.to_string()), + "Function" => functions.push(value.to_string()), + "Variable" => variables.push(value.to_string()), + _ => (), + } + } + + ApiSymbols { + api_version, + headers, + functions, + variables, + } +} + +#[allow(dead_code)] +#[derive(Debug, Deserialize)] +struct SdkOpts { + sdk_symbols: String, + cc_args: String, +} + +/// Load `sdk.opts` file of compiler flags. +fn load_sdk_opts>(path: T) -> SdkOpts { + let file = fs::File::open(path.as_ref()).expect("failed to open sdk.opts"); + + let sdk_opts: SdkOpts = serde_json::from_reader(file).expect("failed to parse sdk.opts JSON"); + + sdk_opts +} + +/// Generate bindings header. +fn generate_bindings_header(api_symbols: &ApiSymbols) -> String { + let mut lines = Vec::new(); + + lines.push(format!( + "#define API_VERSION 0x{:08X}", + api_symbols.api_version + )); + lines.push("#include \"furi/furi.h\"".to_string()); + + for header in &api_symbols.headers { + lines.push(format!("#include \"{header}\"")) + } + + lines.join("\n") +} + +fn main() { + println!("cargo::rerun-if-env-changed=FLIPPER_SDK_VERSION"); + println!("cargo::rerun-if-env-changed=FLIPPER_TOOLCHAIN_VERSION"); + + let out_dir = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let out_dir = Utf8PathBuf::from_path_buf(out_dir).unwrap(); + + let sdk = fetch_sdk(&out_dir).expect("failed to fetch Flipper Zero SDK"); + + if !sdk.is_dir() { + panic!("No such directory: {}", sdk); + } + + let toolchain = fetch_toolchain(&out_dir).expect("failed to fetch Flipper Toolchain"); + + if !toolchain.is_dir() { + panic!("No such directory: {}", toolchain); + } + + let toolchain_include = toolchain.join("arm-none-eabi").join("include"); + + if !toolchain_include.is_dir() { + panic!( + concat!( + "Failed to find toolchain at {:?}.\n", + "You may need to download it first." + ), + toolchain_include + ) + } + + let sdk_headers = sdk.join("sdk_headers"); + + let replace_sdk_root_dir = |s: &str| { + // Need to use '/' on Windows or else include paths won't work + s.replace("SDK_ROOT_DIR", sdk_headers.as_str()) + .replace('\\', "/") + }; + + // Load SDK compiler flags + let sdk_opts = load_sdk_opts(&sdk_headers.join(SDK_OPTS)); + + // Load SDK symbols + let symbols = load_symbols(&sdk_headers.join(&replace_sdk_root_dir(&sdk_opts.sdk_symbols))); + let bindings_header = generate_bindings_header(&symbols); + + // Some of the values are shell-quoted + let cc_flags = shlex::split(&sdk_opts.cc_args).expect("failed to split sdk.opts cc_args"); + let cc_flags: Vec = cc_flags + .into_iter() + .map(|arg| { + match arg.as_str() { + // Force word relocations by disallowing MOVW / MOVT + "-mword-relocations" => String::from("-mno-movt"), + a => replace_sdk_root_dir(a), + } + }) + .collect(); + + // Generate bindings + eprintln!("Generating bindings for SDK {:08X}", symbols.api_version); + let mut bindings = bindgen::builder() + .clang_args(["-target", TARGET]) + .clang_args(["-working-directory", sdk.as_str()]) + .clang_args(["--system-header-prefix=f7_sdk/"]) + .clang_args(["-isystem", toolchain_include.as_str()]) + .clang_args(&cc_flags) + .clang_arg("-Wno-error") + .clang_arg("-fshort-enums") + .clang_arg("-fvisibility=default") + .use_core() + .ctypes_prefix("core::ffi") + .allowlist_var("API_VERSION") + .header_contents("header.h", &bindings_header); + + for function in &symbols.functions { + bindings = bindings.allowlist_function(function); + } + + for variable in &symbols.variables { + bindings = bindings.allowlist_var(variable); + } + + let bindings = match bindings.generate() { + Ok(b) => b, + Err(e) => { + // Separate error output from the preceding clang diag output for legibility + println!("\n{e}"); + panic!("failed to generate bindings") + } + }; + + // `-working-directory` also affects `Bindings::write_to_file` + let outfile = out_dir.join(OUTFILE); + + eprintln!("Writing bindings to {outfile:?}"); + bindings + .write_to_file(outfile) + .expect("failed to write bindings"); +} diff --git a/docs/updating-sdk.md b/docs/updating-sdk.md index 13d950c7..2ee9fccb 100644 --- a/docs/updating-sdk.md +++ b/docs/updating-sdk.md @@ -1,49 +1,18 @@ # Updating the supported SDK -Currently [`flipperzero-sys`] bundles the SDK bindings with each release. +The [`flipperzero-sys`] crate now generates bindings at compile time using a [`build.rs`] script. -This is convenient for users in that it doesn't require any external dependency on -[`flipperzero-firmware`] or compiler toolchain, however it means that each -[`flipperzero-sys`] release is tightly bound to a specific SDK version and thus can only be used -with a specific range of firmware releases. +By default this will fetch the currently supported SDK and toolchain; then use those to generate +the required Rust bindings. -A better approach would be to build the bindings using a `build.rs` script. +The selected SDK and Toolchain versions can be explicitly specified using the following environment variables: -By default this would fetch the currently "supported" SDK and toolchain and use these to generate -bindings. However it should be possible to either override the SDK and toolchain version downloaded -or even point to a local `flipperzero-firmware` checkout. - -## Current process - -Prerequisites: [A recent version of libclang installed](https://rust-lang.github.io/rust-bindgen/requirements.html) - -To update the SDK you require a checkout of [`flipperzero-firmware`] pointing at the target -commit/tag and need to run `./fbt` to download the toolchain and build a local copy of the SDK. -Alternatively you can download a prebuilt SDK from the [Flipper Zero Update Server](https://update.flipperzero.one/builds/firmware/). - -Once the SDK is built, run the [`generate-bindings`] script to build a new [`bindings.rs`]: - -```bash -$ cd tools/ -$ cargo run --bin generate-bindings ../../flipperzero-firmware/build/f7-firmware-D/sdk_headers -$ cp bindings.rs ../crates/sys/src -``` +- `FLIPPER_SDK_VERSION` +- `FLIPPER_TOOLCHAIN_VERSION` +The default values can be found in the `flipperzero-sys` [`build.rs`] script. Make sure to update the SDK details in [`README.md`] before making a new release. -Alternatively, you can generate `binding.rs` in an isolated env using Docker and the following command: - -From the root of the repository, to build the binding for the branch/tag `0.102.3` of the official SDK: - -```shell -image="$(docker build --build-arg BRANCH=0.102.3 --quiet --file tools/Dockerfile .)" -container="$(docker container create --read-only "${image}")" -docker container cp "${container}":bindings.rs ./crates/sys/src/bindings.rs -docker container rm "${container}" -``` - -[`bindings.rs`]: ../crates/sys/src/bindings.rs -[`flipperzero-firmware`]: https://github.com/flipperdevices/flipperzero-firmware +[`build.rs`]: ../crates/sys/build.rs [`flipperzero-sys`]: https://crates.io/crates/flipperzero-sys -[`generate-bindings`]: ../tools/src/bin/generate-bindings.rs [`README.md`]: ../README.md diff --git a/tools/Cargo.lock b/tools/Cargo.lock index e0b3470c..01386530 100644 --- a/tools/Cargo.lock +++ b/tools/Cargo.lock @@ -66,29 +66,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" -[[package]] -name = "bindgen" -version = "0.69.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a00dc851838a2120612785d195287475a3ac45514741da670b735818822129a0" -dependencies = [ - "bitflags 2.6.0", - "cexpr", - "clang-sys", - "itertools", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash", - "shlex", - "syn", - "which 4.4.2", -] - [[package]] name = "bitflags" version = "1.3.2" @@ -122,38 +99,12 @@ version = "1.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50" -[[package]] -name = "camino" -version = "1.1.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b96ec4966b5813e2c0507c1f86115c8c5abaadc3980879c3424042a02fd1ad3" - -[[package]] -name = "cexpr" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" -dependencies = [ - "nom", -] - [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "clang-sys" -version = "1.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" -dependencies = [ - "glob", - "libc", - "libloading", -] - [[package]] name = "clap" version = "4.5.17" @@ -241,27 +192,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "csv" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac574ff4d437a7b5ad237ef331c17ccca63c46479e5b5453eb8e10bb99a759fe" -dependencies = [ - "csv-core", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "csv-core" -version = "0.1.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5efa2b3d7902f4b634a20cae3c9c4e6209dc4779feb6863329607560143efa70" -dependencies = [ - "memchr", -] - [[package]] name = "digest" version = "0.10.7" @@ -272,15 +202,6 @@ dependencies = [ "crypto-common", ] -[[package]] -name = "doxygen-rs" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "415b6ec780d34dcf624666747194393603d0373b7141eef01d12ee58881507d9" -dependencies = [ - "phf", -] - [[package]] name = "either" version = "1.13.0" @@ -313,25 +234,18 @@ checksum = "e8c02a5121d4ea3eb16a80748c74f5549a5665e4c21333c6098f283870fbdea6" name = "flipperzero-tools" version = "0.12.0" dependencies = [ - "bindgen", "bytes", - "camino", "clap", "crossterm", - "csv", - "doxygen-rs", "elf", "hex", "md-5", "once_cell", "rand", "regex", - "serde", - "serde_json", "serialport", - "shlex", "tempfile", - "which 6.0.3", + "which", ] [[package]] @@ -355,12 +269,6 @@ dependencies = [ "wasi", ] -[[package]] -name = "glob" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" - [[package]] name = "heck" version = "0.5.0" @@ -404,49 +312,12 @@ version = "1.70.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" -[[package]] -name = "itertools" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" -dependencies = [ - "either", -] - -[[package]] -name = "itoa" -version = "1.0.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - -[[package]] -name = "lazy_static" -version = "1.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" - -[[package]] -name = "lazycell" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" - [[package]] name = "libc" version = "0.2.158" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" -[[package]] -name = "libloading" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4979f22fdb869068da03c9f7528f8297c6fd2606bc3a4affe42e6a823fdb8da4" -dependencies = [ - "cfg-if", - "windows-targets", -] - [[package]] name = "libudev" version = "0.3.0" @@ -514,12 +385,6 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" -[[package]] -name = "minimal-lexical" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" - [[package]] name = "mio" version = "1.0.2" @@ -544,16 +409,6 @@ dependencies = [ "libc", ] -[[package]] -name = "nom" -version = "7.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" -dependencies = [ - "memchr", - "minimal-lexical", -] - [[package]] name = "once_cell" version = "1.19.0" @@ -583,48 +438,6 @@ dependencies = [ "windows-targets", ] -[[package]] -name = "phf" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" -dependencies = [ - "phf_macros", - "phf_shared", -] - -[[package]] -name = "phf_generator" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" -dependencies = [ - "phf_shared", - "rand", -] - -[[package]] -name = "phf_macros" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" -dependencies = [ - "phf_generator", - "phf_shared", - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "phf_shared" -version = "0.11.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" -dependencies = [ - "siphasher", -] - [[package]] name = "pkg-config" version = "0.3.30" @@ -640,16 +453,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "prettyplease" -version = "0.2.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" -dependencies = [ - "proc-macro2", - "syn", -] - [[package]] name = "proc-macro2" version = "1.0.86" @@ -736,12 +539,6 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" -[[package]] -name = "rustc-hash" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" - [[package]] name = "rustix" version = "0.38.37" @@ -755,50 +552,12 @@ dependencies = [ "windows-sys 0.52.0", ] -[[package]] -name = "ryu" -version = "1.0.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" - [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" -[[package]] -name = "serde" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8e3592472072e6e22e0a54d5904d9febf8508f65fb8552499a1abc7d1078c3a" -dependencies = [ - "serde_derive", -] - -[[package]] -name = "serde_derive" -version = "1.0.210" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "243902eda00fad750862fc144cea25caca5e20d615af0a81bee94ca738f1df1f" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "serde_json" -version = "1.0.128" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ff5456707a1de34e7e37f2a6fd3d3f808c318259cbd01ab6377795054b483d8" -dependencies = [ - "itoa", - "memchr", - "ryu", - "serde", -] - [[package]] name = "serialport" version = "4.5.0" @@ -817,12 +576,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "shlex" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" - [[package]] name = "signal-hook" version = "0.3.17" @@ -853,12 +606,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "0.3.11" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" - [[package]] name = "smallvec" version = "1.13.2" @@ -954,18 +701,6 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" -[[package]] -name = "which" -version = "4.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" -dependencies = [ - "either", - "home", - "once_cell", - "rustix", -] - [[package]] name = "which" version = "6.0.3" diff --git a/tools/Cargo.toml b/tools/Cargo.toml index 1fb14542..0e978abe 100644 --- a/tools/Cargo.toml +++ b/tools/Cargo.toml @@ -6,22 +6,15 @@ version = "0.12.0" edition = "2021" [dependencies] -bindgen = "0.69.1" bytes = "1.5" -camino = "1.1" clap = { version = "4.4", features = ["cargo", "derive"] } crossterm = "0.28" -csv = "1.3" -doxygen-rs = "0.4.2" elf = "0.7.4" hex = "0.4.3" md-5 = "0.10.6" once_cell = "1.19" rand = "0.8.5" regex = "1.10" -serde = { version = "1.0.193", features = ["derive"] } -serde_json = "1.0.108" serialport = "4.3" -shlex = "1.3" tempfile = "3.8" which = "6.0" diff --git a/tools/Dockerfile b/tools/Dockerfile deleted file mode 100644 index ee2cab0f..00000000 --- a/tools/Dockerfile +++ /dev/null @@ -1,64 +0,0 @@ -# syntax=docker/dockerfile:1 -FROM --platform=x86_64 debian:bookworm AS firmware-builder - -# FIRMWARE_GIT should be a git repo with the firmware source code -ARG FIRMWARE_GIT="https://github.com/flipperdevices/flipperzero-firmware.git" -ARG BRANCH="0.102.3" - -# Install dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - ca-certificates \ - git \ - curl \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /app -ADD --keep-git-dir=true "${FIRMWARE_GIT}#${BRANCH}" firmware - -WORKDIR /app/firmware -RUN ./fbt - -#################### -# bindgen -FROM --platform=x86_64 rust:bookworm AS builder - -ARG CLANG_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-17.0.6/clang+llvm-17.0.6-x86_64-linux-gnu-ubuntu-22.04.tar.xz" - -# Install dependencies -RUN apt-get update && apt-get install -y \ - build-essential \ - ca-certificates \ - libudev-dev \ - && rm -rf /var/lib/apt/lists/* - -RUN rustup component add rustfmt - -# We don't use debian's libclang. For details see https://github.com/flipperzero-rs/flipperzero/pull/70#discussion_r1199723419 -RUN mkdir --parents /lib/clang -RUN curl --location "${CLANG_URL}" | tar --extract --xz --directory=/lib/clang --strip-components=1 -ENV LIBCLANG_PATH="/lib/clang/lib" - -COPY --from=firmware-builder /app/firmware/toolchain/ /app/firmware/toolchain/ -COPY --from=firmware-builder /app/firmware/build/f7-firmware-D/sdk_headers/ /app/firmware/build/f7-firmware-D/sdk_headers/ - -WORKDIR /app/flipperzero-rs/ - -# Cache cargo dependencies -WORKDIR /app/flipperzero-rs/tools -COPY ./tools/Cargo.toml ./tools/Cargo.lock ./ -RUN mkdir src && \ - echo "fn main() {}" > src/main.rs && \ - cargo build - -# Now copy the rest of the files -WORKDIR /app/flipperzero-rs/tools -COPY ./tools . -RUN cargo build --bin=generate-bindings -RUN cargo run --bin=generate-bindings /app/firmware/build/f7-firmware-D/sdk_headers - -#################### -FROM scratch -COPY --from=builder /app/flipperzero-rs/tools/bindings.rs / -# Formal entrypoint to simplify the usage of `docker container create` -ENTRYPOINT ["/"] diff --git a/tools/README.md b/tools/README.md index 95addfa9..f08f35a0 100644 --- a/tools/README.md +++ b/tools/README.md @@ -54,7 +54,3 @@ This provides the same interface as [`storage.py`](https://github.com/flipperdev cargo build --release target/release/storage send my-app.fap /ext/apps/Examples/my-app.fap ``` - -## Binding generation - -See [updating-sdk.md](../docs/updating-sdk.md) for details on how to update the SDK bindings using Docker and the `Dockerfile` or locally. diff --git a/tools/src/bin/generate-bindings.rs b/tools/src/bin/generate-bindings.rs deleted file mode 100644 index c8b17fb0..00000000 --- a/tools/src/bin/generate-bindings.rs +++ /dev/null @@ -1,304 +0,0 @@ -//! Generate bindings.rs for Flipper Zero SDK. -//! -//! Usage: `generate-bindings flipperzero-firmware/build/f7-firmware-D/sdk/` - -use std::borrow::Cow; -use std::{env, fs}; - -use bindgen::callbacks::ParseCallbacks; -use camino::{Utf8Path, Utf8PathBuf}; -use clap::{crate_authors, crate_description, crate_version, value_parser}; -use once_cell::sync::Lazy; -use regex::{Captures, Regex, Replacer}; -use serde::Deserialize; - -const TARGET: &str = "thumbv7em-none-eabihf"; -const OUTFILE: &str = "bindings.rs"; -const SDK_OPTS: &str = "sdk.opts"; -#[cfg(all(windows, target_arch = "x86"))] -const TOOLCHAIN: &str = "../../../toolchain/i686-windows/arm-none-eabi/include"; -#[cfg(all(windows, target_arch = "x86_64"))] -const TOOLCHAIN: &str = "../../../toolchain/x86_64-windows/arm-none-eabi/include"; -#[cfg(all(unix, target_arch = "x86"))] -const TOOLCHAIN: &str = "../../../toolchain/i686-linux/arm-none-eabi/include"; -#[cfg(all(unix, target_arch = "x86_64"))] -const TOOLCHAIN: &str = "../../../toolchain/x86_64-linux/arm-none-eabi/include"; -#[cfg(all(target_os = "macos", target_arch = "aarch64"))] -const TOOLCHAIN: &str = "../../../toolchain/x86_64-darwin/arm-none-eabi/include"; -const VISIBILITY_PUBLIC: &str = "+"; - -#[derive(Debug)] -struct ApiSymbols { - pub api_version: u32, - pub headers: Vec, - pub functions: Vec, - pub variables: Vec, -} - -/// Load symbols from `api_symbols.csv`. -fn load_symbols>(path: T) -> ApiSymbols { - let path = path.as_ref(); - - let mut reader = csv::Reader::from_path(path).expect("failed to load symbol file"); - - let mut api_version: u32 = 0; - let mut headers = Vec::new(); - let mut functions = Vec::new(); - let mut variables = Vec::new(); - - for record in reader.records() { - let record = record.expect("failed to parse symbol record"); - let name = &record[0]; - let visibility = &record[1]; - let value = &record[2]; - - if visibility != VISIBILITY_PUBLIC { - continue; - } - - match name { - "Version" => { - let v = value - .split_once('.') - .expect("failed to parse symbol version"); - let major: u16 = v.0.parse().unwrap(); - let minor: u16 = v.1.parse().unwrap(); - - api_version = ((major as u32) << 16) | (minor as u32); - } - "Header" => headers.push(value.to_string()), - "Function" => functions.push(value.to_string()), - "Variable" => variables.push(value.to_string()), - _ => (), - } - } - - ApiSymbols { - api_version, - headers, - functions, - variables, - } -} - -#[allow(dead_code)] -#[derive(Debug, Deserialize)] -struct SdkOpts { - sdk_symbols: String, - cc_args: String, -} - -/// Load `sdk.opts` file of compiler flags. -fn load_sdk_opts>(path: T) -> SdkOpts { - let file = fs::File::open(path.as_ref()).expect("failed to open sdk.opts"); - - let sdk_opts: SdkOpts = serde_json::from_reader(file).expect("failed to parse sdk.opts JSON"); - - sdk_opts -} - -/// Generate bindings header. -fn generate_bindings_header(api_symbols: &ApiSymbols) -> String { - let mut lines = Vec::new(); - - lines.push(format!( - "#define API_VERSION 0x{:08X}", - api_symbols.api_version - )); - lines.push("#include \"furi/furi.h\"".to_string()); - - for header in &api_symbols.headers { - lines.push(format!("#include \"{header}\"")) - } - - lines.join("\n") -} - -/// Parse command-line arguments. -fn parse_args() -> clap::ArgMatches { - clap::Command::new("generate-bindings") - .version(crate_version!()) - .author(crate_authors!()) - .about(crate_description!()) - .arg(clap::Arg::new("sdk").value_parser(value_parser!(Utf8PathBuf))) - .get_matches() -} - -#[derive(Debug)] -struct Cb; - -impl Cb { - fn preprocess_doxygen_comments(comment: &str) -> Cow { - // - static PARAM_IN_OUT: Lazy = Lazy::new(|| { - Regex::new(r"(\n\s*[@\\])param\[(?:\s*(in)\s*,\s*(out)\s*|\s*(out)\s*,\s*(in)\s*)]") - .unwrap() - }); - - struct ParamReplacer; - impl Replacer for ParamReplacer { - fn replace_append(&mut self, caps: &Captures<'_>, dst: &mut String) { - let (prefix, first, second) = (&caps[1], &caps[2], &caps[3]); - dst.reserve(8 + prefix.len() + first.len() + second.len()); - - dst.push_str(prefix); - dst.push_str("param["); - dst.push_str(first); - dst.push(','); - dst.push_str(second); - dst.push(']'); - } - } - - PARAM_IN_OUT.replace_all(comment, ParamReplacer) - } -} - -impl ParseCallbacks for Cb { - fn process_comment(&self, comment: &str) -> Option { - Some(doxygen_rs::transform(&Self::preprocess_doxygen_comments( - comment, - ))) - } -} - -fn main() { - let matches = parse_args(); - - let sdk = matches - .get_one::("sdk") - .expect("failed to find SDK directory"); - - if !sdk.is_dir() { - panic!("No such directory: {}", sdk); - } - - // We must provide absolute paths to Clang. Unfortunately on Windows - // `Path::canonicalize` returns a `\\?\C:\...` style path that is not - // compatible with Clang. - let cwd = Utf8PathBuf::try_from(env::current_dir().unwrap()).unwrap(); - let sdk = cwd.join(sdk); - - let toolchain = sdk.join(TOOLCHAIN); - if !toolchain.is_dir() { - panic!( - concat!( - "Failed to find toolchain at {:?}.\n", - "You may need to download it first." - ), - TOOLCHAIN - ) - } - - let replace_sdk_root_dir = |s: &str| { - // Need to use '/' on Windows, or else include paths don't work - s.replace("SDK_ROOT_DIR", sdk.as_str()).replace('\\', "/") - }; - - // Load SDK compiler flags - let sdk_opts = load_sdk_opts(sdk.join(SDK_OPTS)); - - // Load SDK symbols - let symbols = load_symbols(sdk.join(replace_sdk_root_dir(&sdk_opts.sdk_symbols))); - let bindings_header = generate_bindings_header(&symbols); - - // Some of the values are shell-quoted - let cc_flags = shlex::split(&sdk_opts.cc_args).expect("failed to split sdk.opts cc_args"); - let cc_flags: Vec = cc_flags - .into_iter() - .map(|arg| { - match arg.as_str() { - // Force word relocations by disallowing MOVW / MOVT - "-mword-relocations" => String::from("-mno-movt"), - a => replace_sdk_root_dir(a), - } - }) - .collect(); - - // Generate bindings - eprintln!("Generating bindings for SDK {:08X}", symbols.api_version); - let mut bindings = bindgen::builder() - .clang_args(["-target", TARGET]) - .clang_args(["-working-directory", sdk.as_str()]) - .clang_args(["--system-header-prefix=f7_sdk/"]) - .clang_args(["-isystem", toolchain.as_str()]) - .clang_args(cc_flags) - .clang_arg("-Wno-error") - .clang_arg("-fshort-enums") - .clang_arg("-fvisibility=default") - .use_core() - .parse_callbacks(Box::new(Cb)) - .ctypes_prefix("core::ffi") - .allowlist_var("API_VERSION") - .header_contents("header.h", &bindings_header); - - for function in &symbols.functions { - bindings = bindings.allowlist_function(function); - } - - for variable in &symbols.variables { - bindings = bindings.allowlist_var(variable); - } - - let bindings = match bindings.generate() { - Ok(b) => b, - Err(e) => { - // Separate error output from the preceding clang diag output for legibility - println!("\n{e}"); - panic!("failed to generate bindings") - } - }; - - // `-working-directory` also affects `Bindings::write_to_file` - let outfile = cwd.join(OUTFILE); - - eprintln!("Writing to {OUTFILE:?}"); - bindings - .write_to_file(outfile) - .expect("failed to write bindings"); -} - -#[cfg(test)] -mod tests { - use super::*; - use bindgen::callbacks::ParseCallbacks; - - #[test] - fn doxygen_comments_simple_adhoc_fix() { - let unsupported_comment = "Foo bar baz\n@param[in, out] foo bar baz"; - - let processed_comment = Cb::preprocess_doxygen_comments(unsupported_comment); - - assert_eq!(processed_comment, "Foo bar baz\n@param[in,out] foo bar baz"); - - Cb.process_comment(unsupported_comment) - .expect("The comment should get parsed normally"); - } - - #[test] - fn doxygen_comments_real_life_adhoc_fix() { - let unsupported_comment = " @brief Perform authentication with password. - - Must ONLY be used inside the callback function. - - @param[in, out] instance pointer to the instance to be used in the transaction. - @param[in, out] data pointer to the authentication context. - @return MfUltralightErrorNone on success, an error code on failure."; - - let processed_comment = Cb::preprocess_doxygen_comments(unsupported_comment); - - assert_eq!( - processed_comment, - " @brief Perform authentication with password. - - Must ONLY be used inside the callback function. - - @param[in,out] instance pointer to the instance to be used in the transaction. - @param[in,out] data pointer to the authentication context. - @return MfUltralightErrorNone on success, an error code on failure." - ); - - Cb.process_comment(unsupported_comment) - .expect("The comment should get parsed normally"); - } -}