Gzip artifacts

Co-authored-by: bjorn3 <bjorn3@users.noreply.github.com> Override miniz_oxide to build it with optimizations Building this crate with optimizations decreases the gzipping part of `cargo xtask dist` from `30-40s` down to `3s`, the overhead for `rustc` to apply optimizations is miserable on this background
rust-lang · Jun 21, 2020 · 70da811 · 70da811
1 parent fe25485
commit 70da811
Show file tree

Hide file tree

Showing 6 changed files with 88 additions and 30 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -27,6 +27,11 @@ opt-level = 0
 [profile.release.package.xtask]
 opt-level = 0
 
+# Gzipping the artifacts is up to 10 times faster with optimizations (`cargo xtask dist`).
+# `miniz_ozide` is the direct dependency of `flate2` which does all the heavy lifting
+[profile.dev.package.miniz_oxide]
+opt-level = 3
+
 [patch.'crates-io']
 # rowan = { path = "../rowan" }
 

diff --git a/editors/code/src/main.ts b/editors/code/src/main.ts
@@ -178,7 +178,11 @@ async function bootstrapExtension(config: Config, state: PersistentState): Promi
     assert(!!artifact, `Bad release: ${JSON.stringify(release)}`);
 
     const dest = path.join(config.globalStoragePath, "rust-analyzer.vsix");
-    await download(artifact.browser_download_url, dest, "Downloading rust-analyzer extension");
+    await download({
+        url: artifact.browser_download_url,
+        dest,
+        progressTitle: "Downloading rust-analyzer extension",
+    });
 
     await vscode.commands.executeCommand("workbench.extensions.installExtension", vscode.Uri.file(dest));
     await fs.unlink(dest);
@@ -256,13 +260,13 @@ async function getServer(config: Config, state: PersistentState): Promise<string
     };
     if (config.package.releaseTag === null) return "rust-analyzer";
 
-    let binaryName: string | undefined = undefined;
+    let platform: string | undefined;
     if (process.arch === "x64" || process.arch === "ia32") {
-        if (process.platform === "linux") binaryName = "rust-analyzer-linux";
-        if (process.platform === "darwin") binaryName = "rust-analyzer-mac";
-        if (process.platform === "win32") binaryName = "rust-analyzer-windows.exe";
+        if (process.platform === "linux") platform = "linux";
+        if (process.platform === "darwin") platform = "mac";
+        if (process.platform === "win32") platform = "windows";
     }
-    if (binaryName === undefined) {
+    if (platform === undefined) {
         vscode.window.showErrorMessage(
             "Unfortunately we don't ship binaries for your platform yet. " +
             "You need to manually clone rust-analyzer repository and " +
@@ -273,8 +277,8 @@ async function getServer(config: Config, state: PersistentState): Promise<string
         );
         return undefined;
     }
-
-    const dest = path.join(config.globalStoragePath, binaryName);
+    const ext = platform === "windows" ? ".exe" : "";
+    const dest = path.join(config.globalStoragePath, `rust-analyzer-${platform}${ext}`);
     const exists = await fs.stat(dest).then(() => true, () => false);
     if (!exists) {
         await state.updateServerVersion(undefined);
@@ -291,15 +295,21 @@ async function getServer(config: Config, state: PersistentState): Promise<string
     }
 
     const release = await fetchRelease(config.package.releaseTag);
-    const artifact = release.assets.find(artifact => artifact.name === binaryName);
+    const artifact = release.assets.find(artifact => artifact.name === `rust-analyzer-${platform}.gz`);
     assert(!!artifact, `Bad release: ${JSON.stringify(release)}`);
 
     // Unlinking the exe file before moving new one on its place should prevent ETXTBSY error.
     await fs.unlink(dest).catch(err => {
         if (err.code !== "ENOENT") throw err;
     });
 
-    await download(artifact.browser_download_url, dest, "Downloading rust-analyzer server", { mode: 0o755 });
+    await download({
+        url: artifact.browser_download_url,
+        dest,
+        progressTitle: "Downloading rust-analyzer server",
+        gunzip: true,
+        mode: 0o755
+    });
 
     // Patching executable if that's NixOS.
     if (await fs.stat("/etc/nixos").then(_ => true).catch(_ => false)) {

diff --git a/editors/code/src/net.ts b/editors/code/src/net.ts
@@ -3,6 +3,7 @@ import * as vscode from "vscode";
 import * as stream from "stream";
 import * as fs from "fs";
 import * as os from "os";
+import * as zlib from "zlib";
 import * as path from "path";
 import * as util from "util";
 import { log, assert } from "./util";
@@ -60,22 +61,20 @@ export interface GithubRelease {
     }>;
 }
 
+interface DownloadOpts extends DownloadFileOpts {
+    progressTitle: string;
+}
 
-export async function download(
-    downloadUrl: string,
-    destinationPath: string,
-    progressTitle: string,
-    { mode }: { mode?: number } = {},
-) {
+export async function download(opts: DownloadOpts) {
     await vscode.window.withProgress(
         {
             location: vscode.ProgressLocation.Notification,
             cancellable: false,
-            title: progressTitle
+            title: opts.progressTitle
         },
         async (progress, _cancellationToken) => {
             let lastPercentage = 0;
-            await downloadFile(downloadUrl, destinationPath, mode, (readBytes, totalBytes) => {
+            await downloadFile(opts, (readBytes, totalBytes) => {
                 const newPercentage = (readBytes / totalBytes) * 100;
                 progress.report({
                     message: newPercentage.toFixed(0) + "%",
@@ -88,16 +87,21 @@ export async function download(
     );
 }
 
+interface DownloadFileOpts {
+    url: string;
+    dest: fs.PathLike;
+    mode?: number;
+    gunzip?: boolean;
+}
+
 /**
  * Downloads file from `url` and stores it at `destFilePath` with `mode` (unix permissions).
  * `onProgress` callback is called on recieveing each chunk of bytes
  * to track the progress of downloading, it gets the already read and total
  * amount of bytes to read as its parameters.
  */
 async function downloadFile(
-    url: string,
-    destFilePath: fs.PathLike,
-    mode: number | undefined,
+    { url, dest, mode, gunzip = false }: DownloadFileOpts,
     onProgress: (readBytes: number, totalBytes: number) => void
 ): Promise<void> {
     const res = await fetch(url);
@@ -112,7 +116,7 @@ async function downloadFile(
     const totalBytes = Number(res.headers.get('content-length'));
     assert(!Number.isNaN(totalBytes), "Sanity check of content-length protocol");
 
-    log.debug("Downloading file of", totalBytes, "bytes size from", url, "to", destFilePath);
+    log.debug("Downloading file of", totalBytes, "bytes size from", url, "to", dest);
 
     let readBytes = 0;
     res.body.on("data", (chunk: Buffer) => {
@@ -123,14 +127,15 @@ async function downloadFile(
     // Put the artifact into a temporary folder to prevent partially downloaded files when user kills vscode
     await withTempFile(async tempFilePath => {
         const destFileStream = fs.createWriteStream(tempFilePath, { mode });
-        await pipeline(res.body, destFileStream);
+        const srcStream = gunzip ? res.body.pipe(zlib.createGunzip()) : res.body;
+        await pipeline(srcStream, destFileStream);
         await new Promise<void>(resolve => {
             destFileStream.on("close", resolve);
             destFileStream.destroy();
             // This workaround is awaiting to be removed when vscode moves to newer nodejs version:
             // https://github.com/rust-analyzer/rust-analyzer/issues/3167
         });
-        await moveFile(tempFilePath, destFilePath);
+        await moveFile(tempFilePath, dest);
     });
 }
 

diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml
@@ -14,3 +14,4 @@ pico-args = "0.3.1"
 quote = "1.0.2"
 proc-macro2 = "1.0.8"
 anyhow = "1.0.26"
+flate2 = "1"
diff --git a/xtask/src/dist.rs b/xtask/src/dist.rs
@@ -1,11 +1,16 @@
-use std::path::PathBuf;
+use std::{
+    fs::File,
+    io,
+    path::{Path, PathBuf},
+};
 
 use anyhow::Result;
 
 use crate::{
     not_bash::{date_iso, fs2, pushd, rm_rf, run},
     project_root,
 };
+use flate2::{write::GzEncoder, Compression};
 
 pub fn run_dist(nightly: bool, client_version: Option<String>) -> Result<()> {
     let dist = project_root().join("dist");
@@ -61,18 +66,28 @@ fn dist_server(nightly: bool) -> Result<()> {
         run!("cargo build --manifest-path ./crates/rust-analyzer/Cargo.toml --bin rust-analyzer --release")?;
     }
 
-    let (src, dst) = if cfg!(target_os = "linux") {
-        ("./target/release/rust-analyzer", "./dist/rust-analyzer-linux")
+    let (platform, ext) = if cfg!(target_os = "linux") {
+        ("linux", "")
     } else if cfg!(target_os = "windows") {
-        ("./target/release/rust-analyzer.exe", "./dist/rust-analyzer-windows.exe")
+        ("windows", ".exe")
     } else if cfg!(target_os = "macos") {
-        ("./target/release/rust-analyzer", "./dist/rust-analyzer-mac")
+        ("macos", "")
     } else {
         panic!("Unsupported OS")
     };
 
-    fs2::copy(src, dst)?;
+    let src = format!("./target/release/rust-analyzer{}", ext);
+    let dest = format!("./dist/rust-analyzer-{}.gz", platform);
+
+    gzip(Path::new(&src), Path::new(&dest))?;
+    Ok(())
+}
 
+fn gzip(src_path: &Path, dest_path: &Path) -> Result<()> {
+    let mut encoder = GzEncoder::new(File::create(dest_path)?, Compression::best());
+    let mut input = io::BufReader::new(File::open(src_path)?);
+    io::copy(&mut input, &mut encoder)?;
+    encoder.finish()?;
     Ok(())
 }