Skip to content

Commit

Permalink
feat: allow disabling jlap
Browse files Browse the repository at this point in the history
  • Loading branch information
baszalmstra committed Sep 11, 2023
1 parent 86da1ae commit 281538c
Show file tree
Hide file tree
Showing 3 changed files with 32 additions and 6 deletions.
2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,4 @@ consolidate-commits = true
tag-prefix = ""

[profile.bench]
lto = true
lto = true
17 changes: 15 additions & 2 deletions crates/rattler_repodata_gateway/src/fetch/jlap/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -400,6 +400,7 @@ pub async fn patch_repo_data(

// We already have the latest version; return early because there's nothing to do
if latest_hash == hash {
tracing::info!("The latest hash matches our local data. File up to date.");
return Ok(jlap.get_state(jlap.new_position, new_iv));
}

Expand Down Expand Up @@ -440,11 +441,15 @@ async fn fetch_jlap_with_retry(
client: &AuthenticatedClient,
position: u64,
) -> Result<(Response, u64), JLAPError> {
let range = format!("bytes={}-", position);
tracing::info!("fetching JLAP state from {url} (bytes={position}-)");
let range = format!("bytes={position}-");

match fetch_jlap(url, client, &range).await {
Ok(response) => {
if response.status() == StatusCode::RANGE_NOT_SATISFIABLE && position != 0 {
tracing::warn!(
"JLAP range request could not be satisfied, fetching the entire file.."
);
let range = "bytes=0-";
return match fetch_jlap(url, client, range).await {
Ok(response) => Ok((response, 0)),
Expand Down Expand Up @@ -513,18 +518,25 @@ async fn apply_jlap_patches(
// All these JSON operations are pretty CPU intensive therefor we move them to a blocking task
// to ensure any other async operations will continue to purr along.
let content = match tokio::task::spawn_blocking(move || {
tracing::info!("parsing cached repodata.json as JSON");
let mut doc = match serde_json::from_str(&repo_data_contents) {
Ok(doc) => doc,
Err(error) => return Err(JLAPError::JSONParse(error)),
};

tracing::info!(
"applying patches #{} through #{}",
start_index + 1,
patches.len()
);
// Apply the patches we current have to it
for patch in &patches[start_index..] {
for patch in patches[start_index..].iter() {
if let Err(error) = json_patch::patch(&mut doc, &patch.patch) {
return Err(JLAPError::JSONPatch(error));
}
}

tracing::info!("converting patched JSON back to repodata");
let ordered_doc: OrderedRepoData = match serde_json::from_value(doc) {
Ok(value) => value,
Err(error) => return Err(JLAPError::JSONParse(error)),
Expand Down Expand Up @@ -552,6 +564,7 @@ async fn apply_jlap_patches(
}
};

tracing::info!("writing patched repodata to disk");
match tokio::fs::write(repo_data_path, &content).await {
Ok(_) => Ok(()),
Err(error) => Err(JLAPError::FileSystem(error)),
Expand Down
19 changes: 16 additions & 3 deletions crates/rattler_repodata_gateway/src/fetch/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -139,14 +139,27 @@ impl Variant {
}

/// Additional knobs that allow you to tweak the behavior of [`fetch_repo_data`].
#[derive(Default, Clone)]
#[derive(Clone)]
pub struct FetchRepoDataOptions {
/// How to use the cache. By default it will cache and reuse downloaded repodata.json (if the
/// server allows it).
pub cache_action: CacheAction,

/// Determines which variant to download. See [`Variant`] for more information.
pub variant: Variant,

/// True if repodata can be fetch incrementally using JLAP.
pub jlap_enabled: bool,
}

impl Default for FetchRepoDataOptions {
fn default() -> Self {
Self {
cache_action: Default::default(),
variant: Variant::default(),
jlap_enabled: true,
}
}
}

/// A struct that provides information about download progress.
Expand Down Expand Up @@ -380,7 +393,7 @@ pub async fn fetch_repo_data(

// We first attempt to make a JLAP request; if it fails for any reason, we continue on with
// a normal request.
let jlap_state = if has_jlap && cache_state.is_some() {
let jlap_state = if has_jlap && cache_state.is_some() && options.jlap_enabled {
let repo_data_state = cache_state.as_ref().unwrap();
match jlap::patch_repo_data(
&client,
Expand All @@ -391,7 +404,7 @@ pub async fn fetch_repo_data(
.await
{
Ok(state) => {
tracing::debug!("fetched JLAP patches successfully");
tracing::info!("fetched JLAP patches successfully");
let cache_state = RepoDataState {
blake2_hash: Some(state.footer.latest),
has_zst: variant_availability.has_zst,
Expand Down

0 comments on commit 281538c

Please sign in to comment.