Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor(commands): decouple logic from option structs for check, prune, repair, key, and restore #317

Merged
merged 9 commits into from
Oct 9, 2024
Merged

refactor(commands): decouple logic from option structs for check, prune, repair, key, and restore #317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
124e948
decouple check command
simonsan Oct 7, 2024
ba2cac6
fix tests on windows
simonsan Oct 7, 2024
011fa0b
decouple key command
simonsan Oct 7, 2024
8676e77
decouple repair command
simonsan Oct 7, 2024
d200f7c
decouple restore command
simonsan Oct 7, 2024
fc3d65e
decouple prune command
simonsan Oct 7, 2024
c0e6702
Merge branch 'main' into refactor/decouple_types
simonsan Oct 7, 2024
94aef2b
Merge branch 'main' into refactor/decouple_types
simonsan Oct 7, 2024
40f934f
Merge branch 'main' into refactor/decouple_types
simonsan Oct 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 88 additions & 84 deletions crates/core/src/commands/check.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,106 +136,110 @@ pub struct CheckOptions {
pub read_data_subset: ReadSubsetOption,
}

impl CheckOptions {
/// Runs the `check` command
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to check
///
/// # Errors
///
/// If the repository is corrupted
pub(crate) fn run<P: ProgressBars, S: Open>(
self,
repo: &Repository<P, S>,
trees: Vec<TreeId>,
) -> RusticResult<()> {
let be = repo.dbe();
let cache = repo.cache();
let hot_be = &repo.be_hot;
let raw_be = repo.dbe();
let pb = &repo.pb;
if !self.trust_cache {
if let Some(cache) = &cache {
for file_type in [FileType::Snapshot, FileType::Index] {
// list files in order to clean up the cache
//
// This lists files here and later when reading index / checking snapshots
// TODO: Only list the files once...
_ = be
.list_with_size(file_type)
.map_err(RusticErrorKind::Backend)?;

let p = pb.progress_bytes(format!("checking {file_type:?} in cache..."));
// TODO: Make concurrency (20) customizable
check_cache_files(20, cache, raw_be, file_type, &p)?;
}
/// Runs the `check` command
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to check
/// * `opts` - The check options to use
/// * `trees` - The trees to check
///
/// # Errors
///
/// If the repository is corrupted
///
/// # Panics
///
// TODO: Add panics
pub(crate) fn check_repository<P: ProgressBars, S: Open>(
repo: &Repository<P, S>,
opts: CheckOptions,
trees: Vec<TreeId>,
) -> RusticResult<()> {
let be = repo.dbe();
let cache = repo.cache();
let hot_be = &repo.be_hot;
let raw_be = repo.dbe();
let pb = &repo.pb;
if !opts.trust_cache {
if let Some(cache) = &cache {
for file_type in [FileType::Snapshot, FileType::Index] {
// list files in order to clean up the cache
//
// This lists files here and later when reading index / checking snapshots
// TODO: Only list the files once...
_ = be
.list_with_size(file_type)
.map_err(RusticErrorKind::Backend)?;

let p = pb.progress_bytes(format!("checking {file_type:?} in cache..."));
// TODO: Make concurrency (20) customizable
check_cache_files(20, cache, raw_be, file_type, &p)?;
}
}
}

if let Some(hot_be) = hot_be {
for file_type in [FileType::Snapshot, FileType::Index] {
check_hot_files(raw_be, hot_be, file_type, pb)?;
}
if let Some(hot_be) = hot_be {
for file_type in [FileType::Snapshot, FileType::Index] {
check_hot_files(raw_be, hot_be, file_type, pb)?;
}
}

let index_collector = check_packs(be, hot_be, pb)?;
let index_collector = check_packs(be, hot_be, pb)?;

if let Some(cache) = &cache {
let p = pb.progress_spinner("cleaning up packs from cache...");
let ids: Vec<_> = index_collector
.tree_packs()
.iter()
.map(|(id, size)| (**id, *size))
.collect();
if let Err(err) = cache.remove_not_in_list(FileType::Pack, &ids) {
warn!("Error in cache backend removing pack files: {err}");
}
p.finish();
if let Some(cache) = &cache {
let p = pb.progress_spinner("cleaning up packs from cache...");
let ids: Vec<_> = index_collector
.tree_packs()
.iter()
.map(|(id, size)| (**id, *size))
.collect();
if let Err(err) = cache.remove_not_in_list(FileType::Pack, &ids) {
warn!("Error in cache backend removing pack files: {err}");
}
p.finish();

if !self.trust_cache {
let p = pb.progress_bytes("checking packs in cache...");
// TODO: Make concurrency (5) customizable
check_cache_files(5, cache, raw_be, FileType::Pack, &p)?;
}
if !opts.trust_cache {
let p = pb.progress_bytes("checking packs in cache...");
// TODO: Make concurrency (5) customizable
check_cache_files(5, cache, raw_be, FileType::Pack, &p)?;
}
}

let index_be = GlobalIndex::new_from_index(index_collector.into_index());
let index_be = GlobalIndex::new_from_index(index_collector.into_index());

let packs = check_trees(be, &index_be, trees, pb)?;
let packs = check_trees(be, &index_be, trees, pb)?;

if self.read_data {
let packs = index_be
.into_index()
.into_iter()
.filter(|p| packs.contains(&p.id));
if opts.read_data {
let packs = index_be
.into_index()
.into_iter()
.filter(|p| packs.contains(&p.id));

let packs = self.read_data_subset.apply(packs);
let packs = opts.read_data_subset.apply(packs);

repo.warm_up_wait(packs.iter().map(|pack| pack.id))?;
repo.warm_up_wait(packs.iter().map(|pack| pack.id))?;

let total_pack_size = packs.iter().map(|pack| u64::from(pack.pack_size())).sum();
let p = pb.progress_bytes("reading pack data...");
p.set_length(total_pack_size);
let total_pack_size = packs.iter().map(|pack| u64::from(pack.pack_size())).sum();
let p = pb.progress_bytes("reading pack data...");
p.set_length(total_pack_size);

packs.into_par_iter().for_each(|pack| {
let id = pack.id;
let data = be.read_full(FileType::Pack, &id).unwrap();
match check_pack(be, pack, data, &p) {
Ok(()) => {}
Err(err) => error!("Error reading pack {id} : {err}",),
}
});
p.finish();
}
Ok(())
packs.into_par_iter().for_each(|pack| {
let id = pack.id;
let data = be.read_full(FileType::Pack, &id).unwrap();
match check_pack(be, pack, data, &p) {
Ok(()) => {}
Err(err) => error!("Error reading pack {id} : {err}",),
}
});
p.finish();
}
Ok(())
}

/// Checks if all files in the backend are also in the hot backend
Expand Down
4 changes: 2 additions & 2 deletions crates/core/src/commands/init.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ use crate::{
chunker::random_poly,
commands::{
config::{save_config, ConfigOptions},
key::KeyOptions,
key::{init_key, KeyOptions},
},
crypto::aespoly1305::Key,
error::{RusticErrorKind, RusticResult},
Expand Down Expand Up @@ -86,7 +86,7 @@ pub(crate) fn init_with_config<P, S>(
config: &ConfigFile,
) -> RusticResult<Key> {
repo.be.create().map_err(RusticErrorKind::Backend)?;
let (key, id) = key_opts.init_key(repo, pass)?;
let (key, id) = init_key(repo, key_opts, pass)?;
info!("key {id} successfully added.");
save_config(repo, config.clone(), key)?;

Expand Down
168 changes: 87 additions & 81 deletions crates/core/src/commands/key.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,88 +28,94 @@ pub struct KeyOptions {
pub with_created: bool,
}

impl KeyOptions {
/// Add the current key to the repository.
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to.
/// * `pass` - The password to encrypt the key with.
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized.
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
pub(crate) fn add_key<P, S: Open>(
&self,
repo: &Repository<P, S>,
pass: &str,
) -> RusticResult<KeyId> {
let key = repo.dbe().key();
self.add(repo, pass, *key)
}
/// Add the current key to the repository.
///
/// # Type Parameters
///
/// * `P` - The progress bar type
/// * `S` - The state the repository is in
///
/// # Arguments
///
/// * `repo` - The repository to add the key to
/// * `opts` - The key options to use
/// * `pass` - The password to encrypt the key with
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
pub(crate) fn add_current_key_to_repo<P, S: Open>(
repo: &Repository<P, S>,
opts: &KeyOptions,
pass: &str,
) -> RusticResult<KeyId> {
let key = repo.dbe().key();
add_key_to_repo(repo, opts, pass, *key)
}

/// Initialize a new key.
///
/// # Type Parameters
///
/// * `P` - The progress bar type.
/// * `S` - The state the repository is in.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to.
/// * `pass` - The password to encrypt the key with.
///
/// # Returns
///
/// A tuple of the key and the id of the key.
pub(crate) fn init_key<P, S>(
&self,
repo: &Repository<P, S>,
pass: &str,
) -> RusticResult<(Key, KeyId)> {
// generate key
let key = Key::new();
Ok((key, self.add(repo, pass, key)?))
}
/// Initialize a new key.
///
/// # Type Parameters
///
/// * `P` - The progress bar type
/// * `S` - The state the repository is in
///
/// # Arguments
///
/// * `repo` - The repository to add the key to
/// * `opts` - The key options to use
/// * `pass` - The password to encrypt the key with
///
/// # Returns
///
/// A tuple of the key and the id of the key.
pub(crate) fn init_key<P, S>(
repo: &Repository<P, S>,
opts: &KeyOptions,
pass: &str,
) -> RusticResult<(Key, KeyId)> {
// generate key
let key = Key::new();
Ok((key, add_key_to_repo(repo, opts, pass, key)?))
}

/// Add a key to the repository.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to.
/// * `pass` - The password to encrypt the key with.
/// * `key` - The key to add.
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized.
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
fn add<P, S>(&self, repo: &Repository<P, S>, pass: &str, key: Key) -> RusticResult<KeyId> {
let ko = self.clone();
let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?;
/// Add a key to the repository.
///
/// # Arguments
///
/// * `repo` - The repository to add the key to
/// * `opts` - The key options to use
/// * `pass` - The password to encrypt the key with
/// * `key` - The key to add
///
/// # Errors
///
/// * [`CommandErrorKind::FromJsonError`] - If the key could not be serialized.
///
/// # Returns
///
/// The id of the key.
///
/// [`CommandErrorKind::FromJsonError`]: crate::error::CommandErrorKind::FromJsonError
pub(crate) fn add_key_to_repo<P, S>(
repo: &Repository<P, S>,
opts: &KeyOptions,
pass: &str,
key: Key,
) -> RusticResult<KeyId> {
let ko = opts.clone();
let keyfile = KeyFile::generate(key, &pass, ko.hostname, ko.username, ko.with_created)?;

let data = serde_json::to_vec(&keyfile).map_err(CommandErrorKind::FromJsonError)?;
let id = KeyId::from(hash(&data));
repo.be
.write_bytes(FileType::Key, &id, false, data.into())
.map_err(RusticErrorKind::Backend)?;
Ok(id)
}
let data = serde_json::to_vec(&keyfile).map_err(CommandErrorKind::FromJsonError)?;
let id = KeyId::from(hash(&data));
repo.be
.write_bytes(FileType::Key, &id, false, data.into())
.map_err(RusticErrorKind::Backend)?;
Ok(id)
}
Loading
Loading