build: Use git submodule to download protobuf sources (#1014)

Instead of downloading a tarball, use a submodule to get the protobuf sources. Newer versions of protobuf require recursive submodules for its dependencies

.github/workflows/ci.yml

@@ -58,6 +58,8 @@ jobs:
           - windows-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
       - name: install toolchain (${{ matrix.toolchain }})
         uses: dtolnay/rust-toolchain@master
         with:

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "third_party/protobuf"]
+	path = third_party/protobuf
+	url = https://github.com/protocolbuffers/protobuf.git

protobuf/Cargo.toml

@@ -14,10 +14,7 @@ prost-types = { path = "../prost-types" }
 
 [build-dependencies]
 anyhow = "1.0.1"
-curl = "0.4.13"
-flate2 = "1.0.3"
 prost-build = { path = "../prost-build" }
-tar = "0.4.15"
 tempfile = "3"
 
 # With libz-sys `1.1.8` the msrv has been bumped to 1.54. Since, this dep

protobuf/build.rs

@@ -1,15 +1,9 @@
 use std::env;
 use std::fs;
-use std::io::Cursor;
 use std::path::{Path, PathBuf};
 use std::process::Command;
 
 use anyhow::{ensure, Context, Result};
-use curl::easy::Easy;
-use flate2::bufread::GzDecoder;
-use tar::Archive;
-
-const VERSION: &str = "3.14.0";
 
 static TEST_PROTOS: &[&str] = &[
     "test_messages_proto2.proto",
@@ -41,20 +35,29 @@ static DATASET_PROTOS: &[&str] = &[
 fn main() -> Result<()> {
     let out_dir =
         &PathBuf::from(env::var("OUT_DIR").expect("OUT_DIR environment variable not set"));
-    let protobuf_dir = &out_dir.join(format!("protobuf-{}", VERSION));
+
+    let src_dir = PathBuf::from("../third_party/protobuf");
+    if !src_dir.join("cmake").exists() {
+        anyhow::bail!(
+            "protobuf sources are not checked out; Try `git submodule update --init --recursive`"
+        )
+    }
+
+    let version = git_describe(&src_dir)?;
+    let protobuf_dir = &out_dir.join(format!("protobuf-{}", version));
 
     if !protobuf_dir.exists() {
+        apply_patches(&src_dir)?;
         let tempdir = tempfile::Builder::new()
             .prefix("protobuf")
             .tempdir_in(out_dir)
             .expect("failed to create temporary directory");
 
-        let src_dir = &download_protobuf(tempdir.path())?;
-        let prefix_dir = &src_dir.join("prefix");
+        let prefix_dir = &tempdir.path().join("prefix");
         fs::create_dir(prefix_dir).expect("failed to create prefix directory");
-        install_conformance_test_runner(src_dir, prefix_dir)?;
-        install_protos(src_dir, prefix_dir)?;
-        install_datasets(src_dir, prefix_dir)?;
+        install_conformance_test_runner(&src_dir, prefix_dir)?;
+        install_protos(&src_dir, prefix_dir)?;
+        install_datasets(&src_dir, prefix_dir)?;
         fs::rename(prefix_dir, protobuf_dir).context("failed to move protobuf dir")?;
     }
 
@@ -100,44 +103,26 @@ fn main() -> Result<()> {
     Ok(())
 }
 
-fn download_tarball(url: &str, out_dir: &Path) -> Result<()> {
-    let mut data = Vec::new();
-    let mut handle = Easy::new();
-
-    // Download the tarball.
-    handle.url(url).context("failed to configure tarball URL")?;
-    handle
-        .follow_location(true)
-        .context("failed to configure follow location")?;
-    {
-        let mut transfer = handle.transfer();
-        transfer
-            .write_function(|new_data| {
-                data.extend_from_slice(new_data);
-                Ok(new_data.len())
-            })
-            .context("failed to write download data")?;
-        transfer.perform().context("failed to download tarball")?;
+fn git_describe(src_dir: &Path) -> Result<String> {
+    let output = Command::new("git")
+        .arg("describe")
+        .arg("--tags")
+        .arg("--always")
+        .current_dir(src_dir)
+        .output()
+        .context("Unable to describe protobuf git repo")?;
+    if !output.status.success() {
+        anyhow::bail!(
+            "Unable to describe protobuf git repo: {}",
+            String::from_utf8_lossy(&output.stderr)
+        );
     }
-
-    // Unpack the tarball.
-    Archive::new(GzDecoder::new(Cursor::new(data)))
-        .unpack(out_dir)
-        .context("failed to unpack tarball")
+    let stdout = String::from_utf8_lossy(&output.stdout);
+    Ok(stdout.trim().to_string())
 }
 
-/// Downloads and unpacks a Protobuf release tarball to the provided directory.
-fn download_protobuf(out_dir: &Path) -> Result<PathBuf> {
-    download_tarball(
-        &format!(
-            "https://github.com/google/protobuf/archive/v{}.tar.gz",
-            VERSION
-        ),
-        out_dir,
-    )?;
-    let src_dir = out_dir.join(format!("protobuf-{}", VERSION));
-
-    // Apply patches.
+/// Apply patches to the protobuf source directory
+fn apply_patches(src_dir: &Path) -> Result<()> {
     let mut patch_src = env::current_dir().context("failed to get current working directory")?;
     patch_src.push("src");
     patch_src.push("fix-conformance_test_runner-cmake-build.patch");
@@ -149,9 +134,10 @@ fn download_protobuf(out_dir: &Path) -> Result<PathBuf> {
         .current_dir(&src_dir)
         .status()
         .context("failed to apply patch")?;
-    ensure!(rc.success(), "protobuf patch failed");
+    // exit code: 0 means success; 1 means already applied
+    ensure!(rc.code().unwrap() <= 1, "protobuf patch failed");
 
-    Ok(src_dir)
+    Ok(())
 }
 
 #[cfg(windows)]
@@ -188,7 +174,7 @@ fn install_conformance_test_runner(src_dir: &Path, prefix_dir: &Path) -> Result<
     ensure!(rc.success(), "failed to make protobuf");
 
     // Install the conformance-test-runner binary, since it isn't done automatically.
-    fs::rename(
+    fs::copy(
         src_dir.join("conformance_test_runner"),
         prefix_dir.join("bin").join("conformance-test-runner"),
     )
@@ -204,7 +190,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> {
     let test_include_dir = &include_dir.join("google").join("protobuf");
     fs::create_dir_all(test_include_dir).expect("failed to create test include directory");
     for proto in TEST_PROTOS {
-        fs::rename(
+        fs::copy(
             src_dir
                 .join("src")
                 .join("google")
@@ -219,7 +205,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> {
     let conformance_include_dir = &include_dir.join("conformance");
     fs::create_dir(conformance_include_dir)
         .expect("failed to create conformance include directory");
-    fs::rename(
+    fs::copy(
         src_dir.join("conformance").join("conformance.proto"),
         conformance_include_dir.join("conformance.proto"),
     )
@@ -231,7 +217,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> {
     let datasets_src_dir = &benchmarks_src_dir.join("datasets");
     let datasets_include_dir = &benchmarks_include_dir.join("datasets");
     fs::create_dir(benchmarks_include_dir).expect("failed to create benchmarks include directory");
-    fs::rename(
+    fs::copy(
         benchmarks_src_dir.join("benchmarks.proto"),
         benchmarks_include_dir.join("benchmarks.proto"),
     )
@@ -240,7 +226,7 @@ fn install_protos(src_dir: &Path, prefix_dir: &Path) -> Result<()> {
         let dir = &datasets_include_dir.join(proto.parent().unwrap());
         fs::create_dir_all(dir)
             .with_context(|| format!("unable to create directory {}", dir.display()))?;
-        fs::rename(
+        fs::copy(
             datasets_src_dir.join(proto),
             datasets_include_dir.join(proto),
         )
@@ -262,7 +248,7 @@ fn install_datasets(src_dir: &Path, prefix_dir: &Path) -> Result<()> {
             .join("dataset.google_message1_proto3.pb"),
         Path::new("google_message2").join("dataset.google_message2.pb"),
     ] {
-        fs::rename(
+        fs::copy(
             src_dir.join("benchmarks").join("datasets").join(dataset),
             share_dir.join(dataset.file_name().unwrap()),
         )