[server] Add TLS supprot #1489 (#1517)

* wip

* Update Cargo.toml

* tls support

* tls tests

* tweaks

* Update containerfile

* docker compose

* update docs

* K8s examples

* Update server-bare-metal.mdx

* formatting

* Update agdb_server.yaml

* fix api tests

* Update test_server.rs

* Update agdb_server.yaml

* Update agdb_server.yaml

* Update agdb_server.yaml

.github/workflows/agdb_server.yaml

@@ -24,7 +24,7 @@ jobs:
       - run: cargo clippy -p agdb_server -p agdb_api --all-targets --all-features -- -D warnings
       - uses: taiki-e/install-action@cargo-llvm-cov
       - run: rustup component add llvm-tools-preview
-      - run: cargo llvm-cov -p agdb_server -p agdb_api --all-features --ignore-filename-regex "agdb(.|..)src|agdb_derive" --fail-uncovered-functions 42 --fail-uncovered-lines 231 --show-missing-lines
+      - run: cargo llvm-cov -p agdb_server -p agdb_api --all-features --ignore-filename-regex "agdb(.|..)src|agdb_derive" --fail-uncovered-functions 43 --fail-uncovered-lines 236 --show-missing-lines
 
   agdb_server_image:
     runs-on: ubuntu-latest
@@ -33,10 +33,10 @@ jobs:
       - run: docker build --pull -t agnesoft/agdb:dev -f agdb_server/containerfile .
       - run: docker compose -f agdb_server/compose.yaml up --wait
       - run: sleep 5
-      - run: curl -s http://localhost:3000/studio
-      - run: if [[ "$(curl -s http://localhost:3000/api/v1/cluster/status | grep "\"leader\":true")" == "" ]]; then exit 1; fi
+      - run: curl -s -k https://localhost:3000/studio
+      - run: if [[ "$(curl -s -k https://localhost:3000/api/v1/cluster/status | grep "\"leader\":true")" == "" ]]; then exit 1; fi
       - run: |
-          token=$(curl -X POST http://localhost:3002/api/v1/cluster/user/login -H "Content-Type: application/json" -d '{"username":"admin","password":"admin"}')
-          curl -H "Authorization: Bearer $token" -X POST http://localhost:3002/api/v1/admin/shutdown
-          curl -H "Authorization: Bearer $token" -X POST http://localhost:3000/api/v1/admin/shutdown
-          curl -H "Authorization: Bearer $token" -X POST http://localhost:3001/api/v1/admin/shutdown
+          token=$(curl -X POST -k https://localhost:3002/api/v1/cluster/user/login -H "Content-Type: application/json" -d '{"username":"admin","password":"admin"}')
+          curl -k -H "Authorization: Bearer $token" -X POST https://localhost:3002/api/v1/admin/shutdown
+          curl -k -H "Authorization: Bearer $token" -X POST https://localhost:3000/api/v1/admin/shutdown
+          curl -k -H "Authorization: Bearer $token" -X POST https://localhost:3001/api/v1/admin/shutdown

README.md

@@ -79,6 +79,7 @@ Technical features:
 - Memory mapped for fast querying
 - [Server mode](https://agdb.agnesoft.com/docs/references/server)
 - [Cluster mode](https://agdb.agnesoft.com/docs/references/server#cluster)
+- In-built TLS support
 - [OpenAPI clients](https://agdb.agnesoft.com/api-docs/openapi) in any programming language
 - [Cloud](https://agdb.agnesoft.com/enterprise/cloud) hosted SaaS database
 - _Db itself has no dependencies_
@@ -169,9 +170,17 @@ For database concepts and primitive data types see [concepts](https://agdb.agnes
 
 ### agdb_api
 
-| Feature | Default | Description                                                                                                                                 |
-| ------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
-| reqwest | no      | Enables referential implementation of the `HttpClient` trait for agdb API client using [`reqwest`](https://github.com/seanmonstar/reqwest). |
+| Feature    | Default | Description                                                                                                                                 |
+| ---------- | ------- | ------------------------------------------------------------------------------------------------------------------------------------------- |
+| reqwest    | no      | Enables referential implementation of the `HttpClient` trait for agdb API client using [`reqwest`](https://github.com/seanmonstar/reqwest). |
+| rust-tls   | no      | Enables rust-tls for [`reqwest`](https://github.com/seanmonstar/reqwest).                                                                   |
+| native-tls | no      | Enables native-tls for [`reqwest`](https://github.com/seanmonstar/reqwest).                                                                 |
+
+### agdb_server
+
+| Feature | Default | Description                                                                    |
+| ------- | ------- | ------------------------------------------------------------------------------ |
+| tls     | no      | Enables TLS support via `rustls`. On Windows requires MSVC and CMake to build. |
 
 ## <img width="25" src="https://agdb.agnesoft.com/images/logo.svg" alt="agdb logo">&nbsp;&nbsp;Decision Tree
 

agdb_api/php/ci.sh

@@ -2,8 +2,8 @@ function coverage() {
     rm -f agdb_server.yaml
     rm -rf agdb_server_data
 
-    cargo build --release -p agdb_server
-    cargo run --release -p agdb_server &
+    cargo build -r -p agdb_server
+    cargo run -r -p agdb_server &
     sleep 3
 
     local output

agdb_api/rust/Cargo.toml

@@ -14,11 +14,14 @@ categories = ["database", "api-bindings"]
 [lib]
 
 [features]
+default = []
 reqwest = ["dep:reqwest"]
+rust-tls = ["reqwest/rustls-tls"]
+native-tls = ["reqwest/native-tls"]
 
 [dependencies]
 agdb = { version = "0.10.0", path = "../../agdb", features = ["serde", "openapi"] }
-reqwest = { version = "0.12", features = ["json"], optional = true }
+reqwest = { version = "0.12", default-features = false, features = ["charset", "http2", "macos-system-configuration", "json"], optional = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 utoipa = "5"

agdb_api/typescript/test.sh

@@ -1,7 +1,7 @@
 rm -f agdb_server.yaml
 rm -rf agdb_server_data
-cargo build --release -p agdb_server
-cargo run --release -p agdb_server &
+cargo build -r -p agdb_server
+cargo run -r -p agdb_server &
 
 npx vitest run --coverage
 error_code=$?

agdb_derive/Cargo.toml

@@ -15,6 +15,6 @@ categories = ["database", "database-implementations"]
 proc-macro = true
 
 [dependencies]
-proc-macro2 = "1.0.92"
+proc-macro2 = "1"
 quote = "1"
 syn = "2"

agdb_server/Cargo.toml

@@ -11,15 +11,21 @@ description = "Agnesoft Graph Database Server"
 keywords = ["graph", "database", "api"]
 categories = ["database", "database-implementations"]
 
+[features]
+default = []
+tls = ["dep:axum-server", "dep:rustls", "reqwest/rustls-tls", "agdb_api/rust-tls"]
+
 [dependencies]
 agdb = { version = "0.10.0", path = "../agdb", features = ["serde", "openapi"] }
 agdb_api = { version = "0.10.0", path = "../agdb_api/rust", features = ["reqwest"] }
 axum = { version = "0.8", features = ["http2"] }
 axum-extra = { version = "0.10", features = ["typed-header"] }
+axum-server = { version = "0.7", features = ["tls-rustls"], optional = true }
 include_dir = "0.7"
 http-body-util = "0.1"
-reqwest = { version = "0.12", features = ["json", "stream"] }
+reqwest = { version = "0.12", default-features = false, features = ["charset", "http2", "macos-system-configuration", "json", "stream"] }
 ring = "0.17"
+rustls = { version = "0.23", optional = true }
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 serde_yml = "0.0.12"

agdb_server/compose.yaml

@@ -12,6 +12,12 @@ services:
         target: /agdb/agdb_server.yaml
       - source: pepper
         target: /agdb/pepper
+      - source: cert
+        target: /agdb/cert.pem
+      - source: cert_key
+        target: /agdb/cert.key.pem
+      - source: root_ca
+        target: /agdb/root_ca.pem
   agdb1:
     image: agnesoft/agdb:dev
     hostname: agdb1
@@ -25,6 +31,12 @@ services:
         target: /agdb/agdb_server.yaml
       - source: pepper
         target: /agdb/pepper
+      - source: cert
+        target: /agdb/cert.pem
+      - source: cert_key
+        target: /agdb/cert.key.pem
+      - source: root_ca
+        target: /agdb/root_ca.pem
   agdb2:
     image: agnesoft/agdb:dev
     hostname: agdb2
@@ -38,7 +50,12 @@ services:
         target: /agdb/agdb_server.yaml
       - source: pepper
         target: /agdb/pepper
-
+      - source: cert
+        target: /agdb/cert.pem
+      - source: cert_key
+        target: /agdb/cert.key.pem
+      - source: root_ca
+        target: /agdb/root_ca.pem
 volumes:
   agdb0_data:
   agdb1_data:
@@ -51,39 +68,54 @@ configs:
   agdb0_config:
     content: |
       bind: :::3000
-      address: http://agdb0:3000
+      address: https://agdb0:3000
       basepath: ""
       admin: admin
       log_level: INFO
       data_dir: /agdb/data
       pepper_path: /agdb/pepper
+      tls_certificate: /agdb/cert.pem
+      tls_key: /agdb/cert.key.pem
+      tls_root: /agdb/root_ca.pem
       cluster_token: cluster
       cluster_heartbeat_timeout_ms: 1000
       cluster_term_timeout_ms: 3000
-      cluster: [http://agdb0:3000, http://agdb1:3001, http://agdb2:3002]
+      cluster: [https://agdb0:3000, https://agdb1:3001, https://agdb2:3002]
   agdb1_config:
     content: |
       bind: :::3001
-      address: http://agdb1:3001
+      address: https://agdb1:3001
       basepath: ""
       admin: admin
       log_level: INFO
       data_dir: /agdb/data
       pepper_path: /agdb/pepper
+      tls_certificate: /agdb/cert.pem
+      tls_key: /agdb/cert.key.pem
+      tls_root: /agdb/root_ca.pem
       cluster_token: cluster
       cluster_heartbeat_timeout_ms: 1000
       cluster_term_timeout_ms: 3000
-      cluster: [http://agdb0:3000, http://agdb1:3001, http://agdb2:3002]
+      cluster: [https://agdb0:3000, https://agdb1:3001, https://agdb2:3002]
   agdb2_config:
     content: |
       bind: :::3002
-      address: http://agdb2:3002
+      address: https://agdb2:3002
       basepath: ""
       admin: admin
       log_level: INFO
       data_dir: /agdb/data
       pepper_path: /agdb/pepper
+      tls_certificate: /agdb/cert.pem
+      tls_key: /agdb/cert.key.pem
+      tls_root: /agdb/root_ca.pem
       cluster_token: cluster
       cluster_heartbeat_timeout_ms: 1000
       cluster_term_timeout_ms: 3000
-      cluster: [http://agdb0:3000, http://agdb1:3001, http://agdb2:3002]
+      cluster: [https://agdb0:3000, https://agdb1:3001, https://agdb2:3002]
+  cert:
+    file: ./test_certs/test_cert.pem
+  cert_key:
+    file: ./test_certs/test_cert.key.pem
+  root_ca:
+    file: ./test_certs/test_root_ca.pem

agdb_server/containerfile

@@ -10,8 +10,8 @@ FROM rust:alpine AS builder_server
 WORKDIR /usr/src/agdb_server
 COPY . .
 COPY --from=builder_studio /usr/src/agdb_studio/agdb_studio/dist /usr/src/agdb_server/agdb_studio/dist
-RUN apk add --no-cache musl-dev openssl-dev openssl-libs-static
-RUN cargo build --package agdb_server --release
+RUN apk add --no-cache musl-dev
+RUN cargo build -r -p agdb_server --all-features
 
 FROM alpine:latest
 COPY --from=builder_server /usr/src/agdb_server/target/release/agdb_server /usr/local/bin/agdb_server

agdb_server/src/cluster.rs

@@ -82,6 +82,7 @@ impl ClusterNodeImpl {
         address: &str,
         token: &str,
         responses: UnboundedSender<(Request<ClusterAction>, Response)>,
+        config: &Config,
     ) -> ServerResult<Self> {
         let base = if address.starts_with("http") || address.starts_with("https") {
             address.to_string()
@@ -93,11 +94,7 @@ impl ClusterNodeImpl {
         let base_url = base.trim_end_matches("/").to_string();
 
         Ok(Self {
-            client: ReqwestClient::with_client(
-                reqwest::Client::builder()
-                    .connect_timeout(Duration::from_secs(60))
-                    .build()?,
-            ),
+            client: ReqwestClient::with_client(reqwest_client(config)?),
             url: format!("{base_url}/api/v1/cluster"),
             base_url,
             token: Some(token.to_string()),
@@ -199,6 +196,7 @@ pub(crate) async fn new(config: &Config, db: &ServerDb, db_pool: &DbPool) -> Ser
                 node.as_str(),
                 &config.cluster_token,
                 requests.clone(),
+                config,
             )?));
         }
 
@@ -422,3 +420,40 @@ impl Storage<ClusterAction, ResultNotifier> for ClusterStorage {
         self.db.logs_since(from_index).await
     }
 }
+
+#[cfg(feature = "tls")]
+pub(crate) fn root_ca(config: &Config) -> ServerResult<Option<reqwest::Certificate>> {
+    static ROOT_CA: std::sync::OnceLock<Option<reqwest::Certificate>> = std::sync::OnceLock::new();
+
+    Ok(ROOT_CA
+        .get_or_init(|| {
+            if config.tls_root.is_empty() {
+                return None;
+            }
+
+            let cert_data = std::fs::read(std::path::Path::new(&config.tls_root))
+                .expect("root certificate could not be read");
+            let cert = reqwest::Certificate::from_pem(&cert_data)
+                .expect("root certificate data is invalid");
+            Some(cert)
+        })
+        .clone())
+}
+
+#[cfg(feature = "tls")]
+pub(crate) fn reqwest_client(config: &Config) -> ServerResult<reqwest::Client> {
+    let mut builder = reqwest::Client::builder().timeout(Duration::from_secs(60));
+
+    if let Some(root_ca) = root_ca(config)? {
+        builder = builder.add_root_certificate(root_ca).use_rustls_tls();
+    }
+
+    Ok(builder.build()?)
+}
+
+#[cfg(not(feature = "tls"))]
+pub(crate) fn reqwest_client(_config: &Config) -> ServerResult<reqwest::Client> {
+    Ok(reqwest::Client::builder()
+        .timeout(Duration::from_secs(60))
+        .build()?)
+}

agdb_server/src/config.rs

@@ -22,6 +22,9 @@ pub(crate) struct ConfigImpl {
     pub(crate) log_level: LogLevel,
     pub(crate) data_dir: String,
     pub(crate) pepper_path: String,
+    pub(crate) tls_certificate: String,
+    pub(crate) tls_key: String,
+    pub(crate) tls_root: String,
     pub(crate) cluster_token: String,
     pub(crate) cluster_heartbeat_timeout_ms: u64,
     pub(crate) cluster_term_timeout_ms: u64,
@@ -82,6 +85,9 @@ pub(crate) fn new(config_file: &str) -> ServerResult<Config> {
         log_level: LogLevel(LevelFilter::INFO),
         data_dir: "agdb_server_data".to_string(),
         pepper_path: String::new(),
+        tls_certificate: String::new(),
+        tls_key: String::new(),
+        tls_root: String::new(),
         cluster_token: "cluster".to_string(),
         cluster_heartbeat_timeout_ms: 1000,
         cluster_term_timeout_ms: 3000,
@@ -172,6 +178,9 @@ mod tests {
             log_level: LogLevel(LevelFilter::INFO),
             data_dir: "agdb_server_data".to_string(),
             pepper_path: String::new(),
+            tls_certificate: String::new(),
+            tls_key: String::new(),
+            tls_root: String::new(),
             cluster_token: "cluster".to_string(),
             cluster_heartbeat_timeout_ms: 1000,
             cluster_term_timeout_ms: 3000,
@@ -201,6 +210,9 @@ mod tests {
             log_level: LogLevel(LevelFilter::INFO),
             data_dir: "agdb_server_data".to_string(),
             pepper_path: pepper_file.filename.to_string(),
+            tls_certificate: String::new(),
+            tls_key: String::new(),
+            tls_root: String::new(),
             cluster_token: "cluster".to_string(),
             cluster_heartbeat_timeout_ms: 1000,
             cluster_term_timeout_ms: 3000,
@@ -228,6 +240,9 @@ mod tests {
             log_level: LogLevel(LevelFilter::INFO),
             data_dir: "agdb_server_data".to_string(),
             pepper_path: "missing_file".to_string(),
+            tls_certificate: String::new(),
+            tls_key: String::new(),
+            tls_root: String::new(),
             cluster_token: "cluster".to_string(),
             cluster_heartbeat_timeout_ms: 1000,
             cluster_term_timeout_ms: 3000,
@@ -254,6 +269,9 @@ mod tests {
             log_level: LogLevel(LevelFilter::INFO),
             data_dir: "agdb_server_data".to_string(),
             pepper_path: pepper_file.filename.to_string(),
+            tls_certificate: String::new(),
+            tls_key: String::new(),
+            tls_root: String::new(),
             cluster_token: "cluster".to_string(),
             cluster_heartbeat_timeout_ms: 1000,
             cluster_term_timeout_ms: 3000,