Skip to content
This repository has been archived by the owner on Sep 21, 2024. It is now read-only.

Commit

Permalink
feat: DHT configuration and status API (#207)
Browse files Browse the repository at this point in the history
* Adds `server::APIServer`, a web server for local interaction of name system node.
* Adds `server::HTTPClient` for interfacing with `server::APIServer`.
* `NameSystemClient` trait introduced, implemented by both `NameSystem`
  `and server::HTTPClient`.
* Uses an interim version of libp2p (0.51.0) with libp2p/rust-libp2p#3387 changes, until the major 0.51.0 release.
  • Loading branch information
jsantell authored Jan 31, 2023
1 parent 552a870 commit 7e671cf
Show file tree
Hide file tree
Showing 28 changed files with 1,624 additions and 429 deletions.
206 changes: 137 additions & 69 deletions Cargo.lock

Large diffs are not rendered by default.

2 changes: 2 additions & 0 deletions rust/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ brew install openssl protobuf cmake jq binaryen
**Importantly:** you need an up-to-date Rust toolchain. The most reliable way to
ensure you have this is to follow the instructions on https://rustup.rs/

*Minimum supported rustc version: 1.65.0*

If you wish to compile for targets other than your local platform target, you
can use rustup to get other targets easily. For example:

Expand Down
10 changes: 7 additions & 3 deletions rust/noosphere-ns/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,11 @@ noosphere = { version = "0.5.1", path = "../noosphere", optional = true }
clap = { version = "^4", features = ["derive"], optional = true }
home = { version = "~0.5", optional = true }
toml = { version = "~0.5", optional = true }
libp2p = { version = "0.50.0", default-features = false, features = [ "identify", "dns", "tcp", "tokio", "macros", "noise", "mplex", "yamux", "kad" ] }
# noosphere_ns::server
axum = { version = "~0.5", features = ["json", "headers", "macros"], optional = true }
reqwest = { version = "~0.11", default-features = false, features = ["json", "rustls-tls"], optional = true }
url = { version = "^2", features = [ "serde" ], optional = true }
libp2p = { git = "https://github.com/libp2p/rust-libp2p", rev = "d344406235c779922e6ffec85f0a94181f3efecc", default-features = false, features = [ "identify", "dns", "kad", "macros", "mplex", "noise", "serde", "tcp", "tokio", "yamux" ] }

[dev-dependencies]

Expand All @@ -53,10 +57,10 @@ libipld-cbor = "~0.15"
tempdir = { version = "~0.3" }

[features]
default = ["orb-ns"]
default = ["orb-ns", "api-server"]
api-server = ["axum", "reqwest", "url"]
orb-ns = ["clap", "noosphere", "home", "toml"]


[[bin]]
name = "orb-ns"
required-features = ["orb-ns"]
6 changes: 6 additions & 0 deletions rust/noosphere-ns/src/bin/orb-ns/cli.rs
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,11 @@ pub enum CLICommand {
#[clap(short, long)]
port: Option<u16>,

/// If no configuration path provided, the HTTP listening port of the
/// API web server associated with this DHT node.
#[clap(long)]
api_port: Option<u16>,

/// If no configuration path provided, a list of bootstrap peers to connect to
/// instead of the default bootstrap peers.
#[clap(short, long)]
Expand All @@ -54,6 +59,7 @@ pub struct CLIConfigFile {
pub struct CLIConfigFileNode {
pub key: String,
pub port: Option<u16>,
pub api_port: Option<u16>,
#[serde(default)]
pub peers: Vec<Multiaddr>,
#[serde(default)]
Expand Down
7 changes: 3 additions & 4 deletions rust/noosphere-ns/src/bin/orb-ns/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ mod utils;
#[cfg(not(target_arch = "wasm32"))]
mod inner {
pub use crate::cli::{CLICommand, CLI};
pub use crate::runner::{Runner, RunnerConfig};
pub use crate::runner::{run, RunnerConfig};
pub use anyhow::{anyhow, Result};
pub use clap::Parser;
pub use noosphere::key::{InsecureKeyStorage, KeyStorage};
Expand All @@ -30,9 +30,8 @@ async fn main() -> Result<()> {

match CLI::parse().command {
command @ CLICommand::Run { .. } => {
let mut runner =
Runner::from(RunnerConfig::try_from_command(&key_storage, command).await?);
utils::run_until_abort(runner.run()).await?;
let config = RunnerConfig::try_from_command(&key_storage, command).await?;
utils::run_until_abort(async move { run(config).await }).await?;
Ok(())
}
CLICommand::KeyGen { key } => {
Expand Down
23 changes: 21 additions & 2 deletions rust/noosphere-ns/src/bin/orb-ns/runner/config.rs
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
use std::net::TcpListener;

use crate::cli::{CLICommand, CLIConfigFile, CLIConfigFileNode};
use crate::utils;
use anyhow::{anyhow, Result};
Expand All @@ -10,6 +12,7 @@ use ucan_key_support::ed25519::Ed25519KeyMaterial;
/// resolved data.
pub struct RunnerNodeConfig {
pub key_material: Ed25519KeyMaterial,
pub api_listener: Option<TcpListener>,
pub listening_address: Multiaddr,
pub peers: Vec<Multiaddr>,
pub dht_config: DHTConfig,
Expand All @@ -34,7 +37,14 @@ impl RunnerConfig {
let dht_config = config_node.dht_config;
let port = config_node.port.unwrap_or(0);
let key_material = utils::get_key_material(key_storage, &config_node.key).await?;
let listening_address = utils::create_listening_address(port);
let api_port = if let Some(api_port) = config_node.api_port {
Some(TcpListener::bind(format!("127.0.0.1:{}", api_port))?)
} else {
None
};
let listening_address = format!("/ip4/127.0.0.1/tcp/{}", port)
.parse()
.expect("parseable");
let mut peers = config_node.peers;
if let Some(ref global_peers) = config.peers {
peers.append(&mut global_peers.clone());
Expand All @@ -43,11 +53,12 @@ impl RunnerConfig {
peers.dedup();

nodes.push(RunnerNodeConfig {
api_listener: api_port,
key_material,
listening_address,
peers,
dht_config,
})
});
}
Ok(RunnerConfig::new(nodes))
}
Expand All @@ -62,6 +73,7 @@ impl RunnerConfig {
key,
mut bootstrap,
mut port,
mut api_port,
} => match config {
Some(config_path) => {
let toml_str = tokio::fs::read_to_string(&config_path).await?;
Expand All @@ -82,6 +94,7 @@ impl RunnerConfig {
nodes: vec![CLIConfigFileNode {
key: key_name.clone(),
port: port.take(),
api_port: api_port.take(),
peers,
dht_config: Default::default(),
}],
Expand Down Expand Up @@ -151,6 +164,7 @@ mod tests {
let rc = RunnerConfig::try_from_command(
&env.key_storage,
CLICommand::Run {
api_port: None,
config: None,
key: Some(String::from("single-test-key")),
port: Some(6666),
Expand Down Expand Up @@ -212,6 +226,7 @@ port = 20000
let rc = RunnerConfig::try_from_command(
&env.key_storage,
CLICommand::Run {
api_port: None,
config: env.config_path.to_owned(),
key: None,
port: None,
Expand Down Expand Up @@ -281,24 +296,28 @@ port = 20000

let commands = [
CLICommand::Run {
api_port: None,
config: None,
key: None,
port: Some(6666),
bootstrap: None,
},
CLICommand::Run {
api_port: None,
config: None,
key: Some(String::from("key-does-not-exist")),
port: Some(6666),
bootstrap: None,
},
CLICommand::Run {
api_port: None,
config: env.config_path.to_owned(),
key: None,
port: None,
bootstrap: None,
},
CLICommand::Run {
api_port: None,
config: Some(env.dir_path.join("invalid_path")),
key: None,
port: None,
Expand Down
107 changes: 105 additions & 2 deletions rust/noosphere-ns/src/bin/orb-ns/runner/mod.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,108 @@
mod config;
mod runner;

use anyhow::Result;
pub use config::RunnerConfig;
pub use runner::Runner;
use noosphere_ns::{Multiaddr, NameSystem, NameSystemClient};
use noosphere_storage::{MemoryStorage, SphereDb};
use std::net::TcpListener;
use std::sync::Arc;
use tokio::sync::Mutex;

#[cfg(feature = "api_server")]
mod inner {
use super::*;
pub use noosphere_ns::server::APIServer;

pub fn serve(ns: Arc<Mutex<NameSystem>>, listener: TcpListener) -> APIServer {
APIServer::serve(ns, listener)
}
}
#[cfg(not(feature = "api_server"))]
mod inner {
use super::*;
pub struct APIServer {
_ns: Arc<Mutex<NameSystem>>,
_listener: TcpListener,
}
pub fn serve(ns: Arc<Mutex<NameSystem>>, listener: TcpListener) -> APIServer {
APIServer {
_ns: ns,
_listener: listener,
}
}
}

use inner::*;

struct ActiveNameSystem {
name_system: Arc<Mutex<NameSystem>>,
_api_thread: Option<APIServer>,
}

/// Runner runs one or many DHT nodes based off of provided
/// configuration from a [CLICommand].
pub async fn run(mut config: RunnerConfig) -> Result<()> {
let mut name_systems: Vec<ActiveNameSystem> = vec![];
let mut addresses: Vec<Multiaddr> = vec![];

for node_config in config.nodes.iter_mut() {
let store = SphereDb::new(&MemoryStorage::default()).await?;
let node = NameSystem::new(
&node_config.key_material,
store,
node_config.dht_config.to_owned(),
)?;

// Request address from DHT to resolve default port (0) to
// selected port.
let address = node
.listen(node_config.listening_address.to_owned())
.await?;
node.add_peers(node_config.peers.to_owned()).await?;

println!("Listening on {}...", address);

let wrapped_node = Arc::new(Mutex::new(node));

let api_thread = if let Some(api_listener) = node_config.api_listener.take() {
Some(serve(wrapped_node.clone(), api_listener))
} else {
None
};

name_systems.push(ActiveNameSystem {
name_system: wrapped_node,
_api_thread: api_thread,
});
addresses.push(address);
}

println!("Bootstrapping...");
for (i, ns) in name_systems.iter_mut().enumerate() {
let mut local_peers = addresses.clone();
// Remove a node's own address from peers
// TODO is this necessary?
local_peers.remove(i);

let node = ns.name_system.lock().await;

if local_peers.len() != 0 {
// Add both local peers (other nodes hosted in this process)
// and provided bootstrap peers
node.add_peers(local_peers).await?;
}

node.bootstrap().await?;
}
println!("Bootstrapped.");

let mut tick = tokio::time::interval(tokio::time::Duration::from_secs(1));

loop {
tick.tick().await;
let ns = name_systems.get(0).unwrap().name_system.lock().await;
println!("network check {:#?}", ns.network_info().await?);
}

Ok(())
}
68 changes: 0 additions & 68 deletions rust/noosphere-ns/src/bin/orb-ns/runner/runner.rs

This file was deleted.

7 changes: 0 additions & 7 deletions rust/noosphere-ns/src/bin/orb-ns/utils.rs
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
use anyhow::{anyhow, Result};

use noosphere::key::{InsecureKeyStorage, KeyStorage};
use noosphere_ns::Multiaddr;
use std::future::Future;
use std::path::PathBuf;
use ucan_key_support::ed25519::Ed25519KeyMaterial;
Expand Down Expand Up @@ -36,12 +35,6 @@ pub async fn get_key_material(
}
}

pub fn create_listening_address(port: u16) -> Multiaddr {
format!("/ip4/127.0.0.1/tcp/{}", port)
.parse()
.expect("parseable")
}

pub fn get_keys_dir() -> Result<PathBuf> {
Ok(home::home_dir()
.ok_or_else(|| anyhow!("Could not discover home directory."))?
Expand Down
Loading

0 comments on commit 7e671cf

Please sign in to comment.