Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

1. fix(state): use the new increase_nofile_limit function from rlimit 0.7.0 #3539

Merged
merged 2 commits into from
Feb 21, 2022
Merged

1. fix(state): use the new increase_nofile_limit function from rlimit 0.7.0 #3539

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c047205
fix(state): use the new `increase_nofile_limit` function from rlimit …
teor2345 Feb 15, 2022
9cf0546
Merge branch 'main' into rlimit-0.7.0
mergify[bot] Feb 20, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion zebra-state/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ multiset = { git = "https://github.com/jmitchell/multiset", rev = "91ef8550b518f
proptest = { version = "0.10.1", optional = true }
proptest-derive = { version = "0.3.0", optional = true }
regex = "1"
rlimit = "0.5.4"
rlimit = "0.7.0"
rocksdb = "0.17.0"
serde = { version = "1", features = ["serde_derive"] }
tempfile = "3.3.0"
Expand Down
196 changes: 75 additions & 121 deletions zebra-state/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,9 @@
use serde::{Deserialize, Serialize};
use std::{convert::TryInto, path::PathBuf};

use rlimit::increase_nofile_limit;
use serde::{Deserialize, Serialize};
use tracing::{info, warn};

use zebra_chain::parameters::Network;

/// Configuration for the state service.
Expand Down Expand Up @@ -54,7 +58,7 @@ fn gen_temp_path(prefix: &str) -> PathBuf {

impl Config {
/// The ideal open file limit for Zebra
const IDEAL_OPEN_FILE_LIMIT: usize = 1024;
const IDEAL_OPEN_FILE_LIMIT: u64 = 1024;

/// The minimum number of open files for Zebra to operate normally. Also used
/// as the default open file limit, when the OS doesn't tell us how many
Expand All @@ -65,13 +69,13 @@ impl Config {
/// On Windows, the default limit is 512 high-level I/O files, and 8192
/// low-level I/O files:
/// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
const MIN_OPEN_FILE_LIMIT: usize = 512;
const MIN_OPEN_FILE_LIMIT: u64 = 512;

/// The number of files used internally by Zebra.
///
/// Zebra uses file descriptors for OS libraries (10+), polling APIs (10+),
/// stdio (3), and other OS facilities (2+).
const RESERVED_FILE_COUNT: usize = 48;
const RESERVED_FILE_COUNT: u64 = 48;

/// Returns the path and database options for the finalized state database
pub(crate) fn db_config(&self, network: Network) -> (PathBuf, rocksdb::Options) {
Expand Down Expand Up @@ -100,12 +104,13 @@ impl Config {

let open_file_limit = Config::increase_open_file_limit();
let db_file_limit = Config::get_db_open_file_limit(open_file_limit);

// If the current limit is very large, set the DB limit using the ideal limit
let db_file_limit = db_file_limit.try_into().unwrap_or_else(|_| {
Config::get_db_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT)
.try_into()
.expect("ideal open file limit fits in a config int")
});
let ideal_limit = Config::get_db_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT)
.try_into()
.expect("ideal open file limit fits in a c_int");
let db_file_limit = db_file_limit.try_into().unwrap_or(ideal_limit);

opts.set_max_open_files(db_file_limit);

(path, opts)
Expand All @@ -120,7 +125,7 @@ impl Config {
}

/// Calculate the database's share of `open_file_limit`
fn get_db_open_file_limit(open_file_limit: usize) -> usize {
fn get_db_open_file_limit(open_file_limit: u64) -> u64 {
// Give the DB half the files, and reserve half the files for peers
(open_file_limit - Config::RESERVED_FILE_COUNT) / 2
}
Expand All @@ -136,124 +141,73 @@ impl Config {
/// # Panics
///
/// If the open file limit can not be increased to `MIN_OPEN_FILE_LIMIT`.
#[cfg(unix)]
fn increase_open_file_limit() -> usize {
use rlimit::{getrlimit, Resource};

let (old_limit, hard_rlimit) = match getrlimit(Resource::NOFILE) {
Ok((soft_limit, hard_rlimit)) => (soft_limit.try_into().ok(), Some(hard_rlimit)),
Err(_) => (None, None),
};

// There's no API for reliably setting the soft limit to the lower of the
// hard limit and a desired limit, because:
// * the returned hard limit can be invalid or unrepresentable, and
// * some OS versions (macOS) return larger values than the actual hard
// limit.
// So we try setting the ideal limit, then the minimum limit.
if let Ok(actual_limit) =
Config::set_open_file_limit(Config::IDEAL_OPEN_FILE_LIMIT, hard_rlimit, old_limit)
{
return actual_limit;
}

// Try the hard limit or the minimum, whichever is greater
let min_limit =
if let Some(hard_limit) = hard_rlimit.map(TryInto::try_into).and_then(Result::ok) {
std::cmp::max(Config::MIN_OPEN_FILE_LIMIT, hard_limit)
} else {
Config::MIN_OPEN_FILE_LIMIT
};
if let Ok(actual_limit) = Config::set_open_file_limit(min_limit, hard_rlimit, old_limit) {
tracing::warn!(?actual_limit,
?hard_rlimit,
?old_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the maximum number of open files is below Zebra's ideal limit. Hint: Increase the open file limit to {} before launching Zebra",
Config::IDEAL_OPEN_FILE_LIMIT);
return actual_limit;
}

panic!("open file limit too low: unable to set the number of open files to {}, the minimum number of files required by Zebra. Current soft limit is {:?} and hard limit is {:?}. Hint: Increase the open file limit to {} before launching Zebra",
Config::MIN_OPEN_FILE_LIMIT,
old_limit,
hard_rlimit,
Config::IDEAL_OPEN_FILE_LIMIT);
}

/// Increase the soft open file limit for this process to `new_limit`,
/// and the hard open file limit to `hard_rlimit`.
///
/// If `hard_rlimit` is `None`, also sets the hard limit to `new_limit`.
///
/// If `old_limit` is already greater than or equal to `new_limit`,
/// returns `Ok(old_limit)`.
///
/// Otherwise, tries to set the limit. Returns `Ok(new_limit)` if the
/// limit is set successfully.
#[cfg(unix)]
fn set_open_file_limit(
new_limit: usize,
hard_rlimit: Option<rlimit::Rlim>,
old_limit: Option<usize>,
) -> Result<usize, ()> {
use rlimit::{setrlimit, Resource};
fn increase_open_file_limit() -> u64 {
// `increase_nofile_limit` doesn't do anything on Windows in rlimit 0.7.0.
//
// On Windows, the default limit is:
// - 512 high-level stream I/O files (via the C standard functions), and
// - 8192 low-level I/O files (via the Unix C functions).
// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
//
// If we need more high-level I/O files on Windows,
// use `setmaxstdio` and `getmaxstdio` from the `rlimit` crate:
// https://docs.rs/rlimit/latest/rlimit/#windows
//
// Then panic if `setmaxstdio` fails to set the minimum value,
// and `getmaxstdio` is below the minimum value.

if let Some(old_limit) = old_limit {
if old_limit >= new_limit {
tracing::info!(?new_limit,
current_limit = ?old_limit,
?hard_rlimit,
"the open file limit is at or above the specified limit");
return Ok(old_limit);
// We try setting the ideal limit, then the minimum limit.
let current_limit = match increase_nofile_limit(Config::IDEAL_OPEN_FILE_LIMIT) {
Ok(current_limit) => current_limit,
Err(limit_error) => {
info!(
?limit_error,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"unable to increase the open file limit, \
assuming Zebra can open a minimum number of files"
);

return Config::MIN_OPEN_FILE_LIMIT;
}
}
};

let new_rlimit = new_limit
.try_into()
.expect("new_limit is a valid rlimit value");
if setrlimit(
Resource::NOFILE,
new_rlimit,
hard_rlimit.unwrap_or(new_rlimit),
)
.is_ok()
{
tracing::info!(
?new_limit,
?old_limit,
?hard_rlimit,
"set the open file limit for Zebra"
if current_limit < Config::MIN_OPEN_FILE_LIMIT {
panic!(
"open file limit too low: \
unable to set the number of open files to {}, \
the minimum number of files required by Zebra. \
Current limit is {:?}. \
Hint: Increase the open file limit to {} before launching Zebra",
Config::MIN_OPEN_FILE_LIMIT,
current_limit,
Config::IDEAL_OPEN_FILE_LIMIT
);
} else if current_limit < Config::IDEAL_OPEN_FILE_LIMIT {
warn!(
?current_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the maximum number of open files is below Zebra's ideal limit. \
Hint: Increase the open file limit to {} before launching Zebra",
Config::IDEAL_OPEN_FILE_LIMIT
);
} else if cfg!(windows) {
info!(
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"assuming the open file limit is high enough for Zebra",
);
Ok(new_limit)
} else {
Err(())
info!(
?current_limit,
min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"the open file limit is high enough for Zebra",
);
}
}

/// Assumes that Zebra can open at least the minimum number of files, and
/// returns `MIN_OPEN_FILE_LIMIT`.
///
/// Increasing the open file limit is not yet implemented on Windows. (And
/// other non-unix platforms).
#[cfg(not(unix))]
fn increase_open_file_limit() -> usize {
// On Windows, the default limit is 512 high-level I/O files, and 8192
// low-level I/O files:
// https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio?view=msvc-160#remarks
//
// If we need more high-level I/O files on Windows, we could implement
// support for `setmaxstdio` and `getmaxstdio` in the `rlimit` crate:
// https://github.com/Nugine/rlimit/issues/16#issuecomment-723393017
//
// We should panic if `setmaxstdio` fails to set the minimum value,
// and `getmaxstdio` is below the minimum value.

tracing::info!(min_limit = ?Config::MIN_OPEN_FILE_LIMIT,
ideal_limit = ?Config::IDEAL_OPEN_FILE_LIMIT,
"assuming Zebra can open a minimum number of files");
Config::MIN_OPEN_FILE_LIMIT
current_limit
}
}

Expand Down