Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(commands): decouple logic from option structs for check, prune, repair, key, and restore #317

Merged
merged 9 commits into from
Oct 9, 2024
172 changes: 88 additions & 84 deletions crates/core/src/commands/check.rs
Original file line number Diff line number Diff line change
Expand Up @@ -136,106 +136,110 @@
pub read_data_subset: ReadSubsetOption,
}

impl CheckOptions {
/// Runs the `check` command
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to check
///
/// # Errors
///
/// If the repository is corrupted
pub(crate) fn run<P: ProgressBars, S: Open>(
self,
repo: &Repository<P, S>,
trees: Vec<TreeId>,
) -> RusticResult<()> {
let be = repo.dbe();
let cache = repo.cache();
let hot_be = &repo.be_hot;
let raw_be = repo.dbe();
let pb = &repo.pb;
if !self.trust_cache {
if let Some(cache) = &cache {
for file_type in [FileType::Snapshot, FileType::Index] {
// list files in order to clean up the cache
//
// This lists files here and later when reading index / checking snapshots
// TODO: Only list the files once...
_ = be
.list_with_size(file_type)
.map_err(RusticErrorKind::Backend)?;

let p = pb.progress_bytes(format!("checking {file_type:?} in cache..."));
// TODO: Make concurrency (20) customizable
check_cache_files(20, cache, raw_be, file_type, &p)?;
}
/// Runs the `check` command
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to check
/// * `opts` - The check options to use
/// * `trees` - The trees to check
///
/// # Errors
///
/// If the repository is corrupted
///
/// # Panics
///
// TODO: Add panics
pub(crate) fn check_repository<P: ProgressBars, S: Open>(
repo: &Repository<P, S>,
opts: CheckOptions,
trees: Vec<TreeId>,
) -> RusticResult<()> {
let be = repo.dbe();
let cache = repo.cache();
let hot_be = &repo.be_hot;
let raw_be = repo.dbe();
let pb = &repo.pb;

Check warning on line 168 in crates/core/src/commands/check.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/check.rs#L164-L168

Added lines #L164 - L168 were not covered by tests
if !opts.trust_cache {
if let Some(cache) = &cache {

Check warning on line 170 in crates/core/src/commands/check.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/check.rs#L170

Added line #L170 was not covered by tests
for file_type in [FileType::Snapshot, FileType::Index] {
// list files in order to clean up the cache
//
// This lists files here and later when reading index / checking snapshots
// TODO: Only list the files once...
_ = be
.list_with_size(file_type)

Check warning on line 177 in crates/core/src/commands/check.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/check.rs#L177

Added line #L177 was not covered by tests
.map_err(RusticErrorKind::Backend)?;

let p = pb.progress_bytes(format!("checking {file_type:?} in cache..."));
// TODO: Make concurrency (20) customizable
check_cache_files(20, cache, raw_be, file_type, &p)?;
}
}
}

if let Some(hot_be) = hot_be {
for file_type in [FileType::Snapshot, FileType::Index] {
check_hot_files(raw_be, hot_be, file_type, pb)?;
}
if let Some(hot_be) = hot_be {
for file_type in [FileType::Snapshot, FileType::Index] {
check_hot_files(raw_be, hot_be, file_type, pb)?;

Check warning on line 189 in crates/core/src/commands/check.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/check.rs#L188-L189

Added lines #L188 - L189 were not covered by tests
}
}

let index_collector = check_packs(be, hot_be, pb)?;
let index_collector = check_packs(be, hot_be, pb)?;

if let Some(cache) = &cache {
let p = pb.progress_spinner("cleaning up packs from cache...");
let ids: Vec<_> = index_collector
.tree_packs()
.iter()
.map(|(id, size)| (**id, *size))
.collect();
if let Err(err) = cache.remove_not_in_list(FileType::Pack, &ids) {
warn!("Error in cache backend removing pack files: {err}");
}
p.finish();
if let Some(cache) = &cache {
let p = pb.progress_spinner("cleaning up packs from cache...");
let ids: Vec<_> = index_collector
.tree_packs()
.iter()
.map(|(id, size)| (**id, *size))
.collect();
if let Err(err) = cache.remove_not_in_list(FileType::Pack, &ids) {
warn!("Error in cache backend removing pack files: {err}");

Check warning on line 203 in crates/core/src/commands/check.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/check.rs#L203

Added line #L203 was not covered by tests
}
p.finish();

if !self.trust_cache {
let p = pb.progress_bytes("checking packs in cache...");
// TODO: Make concurrency (5) customizable
check_cache_files(5, cache, raw_be, FileType::Pack, &p)?;
}
if !opts.trust_cache {
let p = pb.progress_bytes("checking packs in cache...");
// TODO: Make concurrency (5) customizable
check_cache_files(5, cache, raw_be, FileType::Pack, &p)?;
}
}

let index_be = GlobalIndex::new_from_index(index_collector.into_index());
let index_be = GlobalIndex::new_from_index(index_collector.into_index());

let packs = check_trees(be, &index_be, trees, pb)?;
let packs = check_trees(be, &index_be, trees, pb)?;

if self.read_data {
let packs = index_be
.into_index()
.into_iter()
.filter(|p| packs.contains(&p.id));
if opts.read_data {
let packs = index_be
.into_index()
.into_iter()
.filter(|p| packs.contains(&p.id));

let packs = self.read_data_subset.apply(packs);
let packs = opts.read_data_subset.apply(packs);

Check warning on line 224 in crates/core/src/commands/check.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/check.rs#L224

Added line #L224 was not covered by tests

repo.warm_up_wait(packs.iter().map(|pack| pack.id))?;
repo.warm_up_wait(packs.iter().map(|pack| pack.id))?;

let total_pack_size = packs.iter().map(|pack| u64::from(pack.pack_size())).sum();
let p = pb.progress_bytes("reading pack data...");
p.set_length(total_pack_size);
let total_pack_size = packs.iter().map(|pack| u64::from(pack.pack_size())).sum();
let p = pb.progress_bytes("reading pack data...");
p.set_length(total_pack_size);

packs.into_par_iter().for_each(|pack| {
let id = pack.id;
let data = be.read_full(FileType::Pack, &id).unwrap();
match check_pack(be, pack, data, &p) {
Ok(()) => {}
Err(err) => error!("Error reading pack {id} : {err}",),
}
});
p.finish();
}
Ok(())
packs.into_par_iter().for_each(|pack| {
let id = pack.id;
let data = be.read_full(FileType::Pack, &id).unwrap();
match check_pack(be, pack, data, &p) {
Ok(()) => {}
Err(err) => error!("Error reading pack {id} : {err}",),

Check warning on line 237 in crates/core/src/commands/check.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/check.rs#L236-L237

Added lines #L236 - L237 were not covered by tests
}
});
p.finish();
}
Ok(())
}

/// Checks if all files in the backend are also in the hot backend
Expand Down
4 changes: 2 additions & 2 deletions crates/core/src/commands/init.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use crate::{
chunker::random_poly,
commands::{
config::{save_config, ConfigOptions},
key::KeyOptions,
key::{init_key, KeyOptions},
},
crypto::aespoly1305::Key,
error::{RusticErrorKind, RusticResult},
Expand Down Expand Up @@ -86,7 +86,7 @@ pub(crate) fn init_with_config<P, S>(
config: &ConfigFile,
) -> RusticResult<Key> {
repo.be.create().map_err(RusticErrorKind::Backend)?;
let (key, id) = key_opts.init_key(repo, pass)?;
let (key, id) = init_key(repo, key_opts, pass)?;
info!("key {id} successfully added.");
save_config(repo, config.clone(), key)?;

Expand Down
168 changes: 87 additions & 81 deletions crates/core/src/commands/key.rs
Original file line number Diff line number Diff line change
Expand Up @@ -28,88 +28,94 @@
pub with_created: bool,
}

impl KeyOptions {
/// Add the current key to the repository.
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to.
/// * `pass` - The password to encrypt the key with.
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized.
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
pub(crate) fn add_key<P, S: Open>(
&self,
repo: &Repository<P, S>,
pass: &str,
) -> RusticResult<KeyId> {
let key = repo.dbe().key();
self.add(repo, pass, *key)
}
/// Add the current key to the repository.
///
/// # Type Parameters
///
/// * `P` - The progress bar type
/// * `S` - The state the repository is in
///
/// # Arguments
///
/// * `repo` - The repository to add the key to
/// * `opts` - The key options to use
/// * `pass` - The password to encrypt the key with
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
pub(crate) fn add_current_key_to_repo<P, S: Open>(
repo: &Repository<P, S>,
opts: &KeyOptions,
pass: &str,
) -> RusticResult<KeyId> {
let key = repo.dbe().key();
add_key_to_repo(repo, opts, pass, *key)

Check warning on line 59 in crates/core/src/commands/key.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/key.rs#L58-L59

Added lines #L58 - L59 were not covered by tests
}

/// Initialize a new key.
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to.
/// * `pass` - The password to encrypt the key with.
///
/// # Returns
///
/// A tuple of the key and the id of the key.
pub(crate) fn init_key<P, S>(
&self,
repo: &Repository<P, S>,
pass: &str,
) -> RusticResult<(Key, KeyId)> {
// generate key
let key = Key::new();
Ok((key, self.add(repo, pass, key)?))
}
/// Initialize a new key.
///
/// # Type Parameters
///
/// * `P` - The progress bar type
/// * `S` - The state the repository is in
///
/// # Arguments
///
/// * `repo` - The repository to add the key to
/// * `opts` - The key options to use
/// * `pass` - The password to encrypt the key with
///
/// # Returns
///
/// A tuple of the key and the id of the key.
pub(crate) fn init_key<P, S>(

Check warning on line 78 in crates/core/src/commands/key.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/key.rs#L78

Added line #L78 was not covered by tests
repo: &Repository<P, S>,
opts: &KeyOptions,
pass: &str,
) -> RusticResult<(Key, KeyId)> {
// generate key
let key = Key::new();
Ok((key, add_key_to_repo(repo, opts, pass, key)?))
}

/// Add a key to the repository.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to.
/// * `pass` - The password to encrypt the key with.
/// * `key` - The key to add.
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized.
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
fn add<P, S>(&self, repo: &Repository<P, S>, pass: &str, key: Key) -> RusticResult<KeyId> {
let ko = self.clone();
let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?;
/// Add a key to the repository.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to
/// * `opts` - The key options to use
/// * `pass` - The password to encrypt the key with
/// * `key` - The key to add
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized.
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
pub(crate) fn add_key_to_repo<P, S>(
repo: &Repository<P, S>,
opts: &KeyOptions,
pass: &str,
key: Key,
) -> RusticResult<KeyId> {
let ko = opts.clone();

Check warning on line 112 in crates/core/src/commands/key.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/key.rs#L112

Added line #L112 was not covered by tests
let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?;

let data = serde_json::to_vec(&keyfile).map_err(CommandErrorKind::FromJsonError)?;
let id = KeyId::from(hash(&data));
repo.be
.write_bytes(FileType::Key, &id, false, data.into())
.map_err(RusticErrorKind::Backend)?;
Ok(id)
}
let data = serde_json::to_vec(&keyfile).map_err(CommandErrorKind::FromJsonError)?;
let id = KeyId::from(hash(&data));
repo.be
.write_bytes(FileType::Key, &id, false, data.into())
.map_err(RusticErrorKind::Backend)?;

Check warning on line 119 in crates/core/src/commands/key.rs

View check run for this annotation

Codecov / codecov/patch

crates/core/src/commands/key.rs#L119

Added line #L119 was not covered by tests
Ok(id)
}
Loading