Skip to content

Commit

Permalink
2. refactor(state): move all RocksDB API calls to the disk_db module (#…
Browse files Browse the repository at this point in the history 
…3578)

* refactor(state): move RocksDB-specific initialization to a new module

* refactor(state): move RocksDB-specific shutdown to a new module

* refactor(state): temporarily allow RocksDB-specific reads and writes, without a new module

Unlike the last few commits, this one actually compiles.

* refactor(state): add a DiskWriteBatch wrapper for RocksDB writes

* refactor(state): move finalized state test methods to a test module
  • Loading branch information
@teor2345
teor2345 authored Feb 22, 2022
1 parent 8e36686 commit 32017f9
Show file tree
Hide file tree
Showing 6 changed files with 433 additions and 334 deletions.
136 changes: 5 additions & 131 deletions zebra-state/src/config.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
use std::{convert::TryInto, path::PathBuf};
use std::path::PathBuf;

use rlimit::increase_nofile_limit;
use serde::{Deserialize, Serialize};
use tracing::{info, warn};

use zebra_chain::parameters::Network;

Expand Down Expand Up @@ -57,34 +55,14 @@ fn gen_temp_path(prefix: &str) -> PathBuf {
}

impl Config {
/// The ideal open file limit for Zebra
const IDEAL_OPEN_FILE_LIMIT: u64 = 1024;

/// The minimum number of open files for Zebra to operate normally. Also used
/// as the default open file limit, when the OS doesn't tell us how many
/// files we can use.
///
/// We want 100+ file descriptors for peers, and 100+ for the database.
///
/// On Windows, the default limit is 512 high-level I/O files, and 8192
/// low-level I/O files:
/// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
const MIN_OPEN_FILE_LIMIT: u64 = 512;

/// The number of files used internally by Zebra.
///
/// Zebra uses file descriptors for OS libraries (10+), polling APIs (10+),
/// stdio (3), and other OS facilities (2+).
const RESERVED_FILE_COUNT: u64 = 48;

/// Returns the path and database options for the finalized state database
pub(crate) fn db_config(&self, network: Network) -> (PathBuf, rocksdb::Options) {
/// Returns the path for the finalized state database
pub(crate) fn db_path(&self, network: Network) -> PathBuf {
let net_dir = match network {
Network::Mainnet => "mainnet",
Network::Testnet => "testnet",
};

let path = if self.ephemeral {
if self.ephemeral {
gen_temp_path(&format!(
"zebra-state-v{}-{}",
crate::constants::DATABASE_FORMAT_VERSION,
Expand All @@ -95,25 +73,7 @@ impl Config {
.join("state")
.join(format!("v{}", crate::constants::DATABASE_FORMAT_VERSION))
.join(net_dir)
};

let mut opts = rocksdb::Options::default();

opts.create_if_missing(true);
opts.create_missing_column_families(true);

let open_file_limit = Config::increase_open_file_limit();
let db_file_limit = Config::get_db_open_file_limit(open_file_limit);

// If the current limit is very large, set the DB limit using the ideal limit
let ideal_limit = Config::get_db_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT)
.try_into()
.expect("ideal open file limit fits in a c_int");
let db_file_limit = db_file_limit.try_into().unwrap_or(ideal_limit);

opts.set_max_open_files(db_file_limit);

(path, opts)
}
}

/// Construct a config for an ephemeral database
Expand All @@ -123,92 +83,6 @@ impl Config {
..Config::default()
}
}

/// Calculate the database's share of `open_file_limit`
fn get_db_open_file_limit(open_file_limit: u64) -> u64 {
// Give the DB half the files, and reserve half the files for peers
(open_file_limit - Config::RESERVED_FILE_COUNT) / 2
}

/// Increase the open file limit for this process to `IDEAL_OPEN_FILE_LIMIT`.
/// If that fails, try `MIN_OPEN_FILE_LIMIT`.
///
/// If the current limit is above `IDEAL_OPEN_FILE_LIMIT`, leaves it
/// unchanged.
///
/// Returns the current limit, after any successful increases.
///
/// # Panics
///
/// If the open file limit can not be increased to `MIN_OPEN_FILE_LIMIT`.
fn increase_open_file_limit() -> u64 {
// `increase_nofile_limit` doesn't do anything on Windows in rlimit 0.7.0.
//
// On Windows, the default limit is:
// - 512 high-level stream I/O files (via the C standard functions), and
// - 8192 low-level I/O files (via the Unix C functions).
// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
//
// If we need more high-level I/O files on Windows,
// use `setmaxstdio` and `getmaxstdio` from the `rlimit` crate:
// https://docs.rs/rlimit/latest/rlimit/#windows
//
// Then panic if `setmaxstdio` fails to set the minimum value,
// and `getmaxstdio` is below the minimum value.

// We try setting the ideal limit, then the minimum limit.
let current_limit = match increase_nofile_limit(Config::IDEAL_OPEN_FILE_LIMIT) {
Ok(current_limit) => current_limit,
Err(limit_error) => {
info!(
?limit_error,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"unable to increase the open file limit, \
assuming Zebra can open a minimum number of files"
);

return Config::MIN_OPEN_FILE_LIMIT;
}
};

if current_limit < Config::MIN_OPEN_FILE_LIMIT {
panic!(
"open file limit too low: \
unable to set the number of open files to {}, \
the minimum number of files required by Zebra. \
Current limit is {:?}. \
Hint: Increase the open file limit to {} before launching Zebra",
Config::MIN_OPEN_FILE_LIMIT,
current_limit,
Config::IDEAL_OPEN_FILE_LIMIT
);
} else if current_limit < Config::IDEAL_OPEN_FILE_LIMIT {
warn!(
?current_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the maximum number of open files is below Zebra's ideal limit. \
Hint: Increase the open file limit to {} before launching Zebra",
Config::IDEAL_OPEN_FILE_LIMIT
);
} else if cfg!(windows) {
info!(
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"assuming the open file limit is high enough for Zebra",
);
} else {
info!(
?current_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the open file limit is high enough for Zebra",
);
}

current_limit
}
}

impl Default for Config {
Expand Down
3 changes: 3 additions & 0 deletions zebra-state/src/lib.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,9 @@
#![doc(html_logo_url = "https://www.zfnd.org/images/zebra-icon.png")]
#![doc(html_root_url = "https://doc.zebra.zfnd.org/zebra_state")]

#[macro_use]
extern crate tracing;

#[cfg(any(test, feature = "proptest-impl"))]
mod arbitrary;
mod config;
Expand Down
Loading

0 comments on commit 32017f9

Please sign in to comment.