From e3a7ebf4e7bdcf9d84288592638146d5c389deb7 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Fri, 10 Jan 2025 18:42:16 +0800 Subject: [PATCH] local cache for rpc test snapshots --- Cargo.lock | 7 +++ Cargo.toml | 1 + src/tool/subcommands/api_cmd/test_snapshot.rs | 53 +++++++++++++++---- 3 files changed, 50 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c68eb0085c6..55313130a1e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3097,6 +3097,7 @@ dependencies = [ "libp2p-swarm-test", "libsecp256k1", "lru", + "md5", "memmap2 0.9.5", "memory-stats", "mimalloc", @@ -5939,6 +5940,12 @@ dependencies = [ "rawpointer", ] +[[package]] +name = "md5" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" + [[package]] name = "memchr" version = "2.7.4" diff --git a/Cargo.toml b/Cargo.toml index f428f4d9de8..764b53c1f91 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -238,6 +238,7 @@ glob = "0.3" http-range-header = "0.4" insta = { version = "1", features = ["yaml"] } libp2p-swarm-test = { workspace = true } +md5 = "0.7" num-bigint = { version = "0.4", features = ['quickcheck'] } petgraph = "0.7" predicates = "3" diff --git a/src/tool/subcommands/api_cmd/test_snapshot.rs b/src/tool/subcommands/api_cmd/test_snapshot.rs index 3f0175e506c..aeed098b48c 100644 --- a/src/tool/subcommands/api_cmd/test_snapshot.rs +++ b/src/tool/subcommands/api_cmd/test_snapshot.rs @@ -143,7 +143,8 @@ async fn ctx( #[cfg(test)] mod tests { use super::*; - use crate::daemon::db_util::download_to; + use crate::{daemon::db_util::download_to, utils::net::global_http_client}; + use directories::ProjectDirs; use itertools::Itertools as _; use url::Url; @@ -158,19 +159,49 @@ mod tests { .as_str(), ) .ok() + .map(|url| (n, url)) }) .collect_vec(); - for url in urls { - print!("Testing {url} ..."); - let tmp_dir = tempfile::tempdir().unwrap(); - let tmp = tempfile::NamedTempFile::new_in(&tmp_dir) - .unwrap() - .into_temp_path(); - println!("start downloading at {}", tmp.display()); - download_to(&url, &tmp).await.unwrap(); - println!("done downloading {}", tmp.display()); - run_test_from_snapshot(&tmp).await.unwrap(); + let project_dir = ProjectDirs::from("com", "ChainSafe", "Forest").unwrap(); + let cache_dir = project_dir.cache_dir().join("test").join("rpc-snapshots"); + for (filename, url) in urls { + let cache_file_path = cache_dir.join(filename); + let is_file_cached = match get_file_md5_etag(&cache_file_path) { + Some(file_etag) => { + let url_etag = get_digital_ocean_space_url_etag(url.clone()).await.unwrap(); + if Some(&file_etag) == url_etag.as_ref() { + true + } else { + println!( + "etag mismatch, file: {filename}, local: {file_etag}, remote: {}", + url_etag.unwrap_or_default() + ); + false + } + } + None => false, + }; + if !is_file_cached { + println!("Downloading from {url} to {}", cache_file_path.display()); + download_to(&url, &cache_file_path).await.unwrap(); + } + print!("Testing {filename} ..."); + run_test_from_snapshot(&cache_file_path).await.unwrap(); println!(" succeeded."); } } + + async fn get_digital_ocean_space_url_etag(url: Url) -> anyhow::Result> { + let response = global_http_client().head(url).send().await?; + Ok(response + .headers() + .get("etag") + .and_then(|v| v.to_str().ok().map(|v| v.replace('"', "").to_string()))) + } + + fn get_file_md5_etag(path: &Path) -> Option { + std::fs::read(path) + .ok() + .map(|bytes| format!("{:x}", md5::compute(bytes.as_slice()))) + } }