gramineproject · dimakuv · Oct 28, 2022 · Oct 17, 2022
diff --git a/CI-Examples/bash/manifest.template b/CI-Examples/bash/manifest.template
@@ -21,7 +21,7 @@ fs.mounts = [
 sgx.debug = true
 sgx.nonpie_binary = true
 sgx.enclave_size = "512M"
-sgx.thread_num = 4
+sgx.max_threads = 4
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/CI-Examples/blender/blender.manifest.template b/CI-Examples/blender/blender.manifest.template
@@ -24,7 +24,7 @@ sgx.debug = true
 sgx.nonpie_binary = true
 sys.stack.size = "8M"
 sgx.enclave_size = "2048M"
-sgx.thread_num = 64
+sgx.max_threads = 64
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/CI-Examples/lighttpd/lighttpd.manifest.template b/CI-Examples/lighttpd/lighttpd.manifest.template
@@ -22,7 +22,7 @@ fs.mounts = [
 sgx.debug = true
 sgx.nonpie_binary = true
 sgx.enclave_size = "256M"
-sgx.thread_num = 3
+sgx.max_threads = 3
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/CI-Examples/memcached/memcached.manifest.template b/CI-Examples/memcached/memcached.manifest.template
@@ -25,7 +25,7 @@ fs.mounts = [
 
 sgx.debug = true
 sgx.nonpie_binary = true
-sgx.thread_num = 16
+sgx.max_threads = 16
 
 # Memcached does not fail explicitly when enclave memory is exhausted. Instead, Memcached goes into
 # infinite loop without a listening socket. You can trigger this incorrect behavior by increasing

diff --git a/CI-Examples/nginx/nginx.manifest.template b/CI-Examples/nginx/nginx.manifest.template
@@ -28,7 +28,7 @@ fs.mounts = [
 sgx.debug = true
 sgx.nonpie_binary = true
 sgx.enclave_size = "512M"
-sgx.thread_num = 4
+sgx.max_threads = 4
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/CI-Examples/python/python.manifest.template b/CI-Examples/python/python.manifest.template
@@ -30,7 +30,7 @@ sys.enable_extra_runtime_domain_names_conf = true
 sgx.debug = true
 sgx.nonpie_binary = true
 sgx.enclave_size = "512M"
-sgx.thread_num = 32
+sgx.max_threads = 32
 
 sgx.remote_attestation = "{{ ra_type }}"
 sgx.ra_client_spid = "{{ ra_client_spid }}"

diff --git a/CI-Examples/ra-tls-mbedtls/client.manifest.template b/CI-Examples/ra-tls-mbedtls/client.manifest.template
@@ -26,7 +26,7 @@ sys.enable_extra_runtime_domain_names_conf = true
 
 sgx.debug = true
 sgx.enclave_size = "512M"
-sgx.thread_num = 4
+sgx.max_threads = 4
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/CI-Examples/redis/redis-server.manifest.template b/CI-Examples/redis/redis-server.manifest.template
@@ -78,7 +78,7 @@ sgx.enclave_size = "1024M"
 # and one for asynchronous events/alarms. Redis is technically single-threaded
 # but spawns couple additional threads to do background bookkeeping. Therefore,
 # specifying '8' allows to run a maximum of 6 Redis threads which is enough.
-sgx.thread_num = 8
+sgx.max_threads = 8
 
 # Redis executable is typically a PIE (Position Independent Executable) on most
 # modern OS distros (e.g., Ubuntu 18.04). However, on some OS distros (notably,

diff --git a/CI-Examples/rust/rust-hyper-http-server.manifest.template b/CI-Examples/rust/rust-hyper-http-server.manifest.template
@@ -40,4 +40,4 @@ sys.insecure__allow_eventfd = true
 # - any threads and threadpools you might be starting
 # - helper threads internal to Gramine — see:
 #   https://gramine.readthedocs.io/en/latest/manifest-syntax.html#number-of-threads
-sgx.thread_num = 8
+sgx.max_threads = 8
diff --git a/CI-Examples/rust/src/main.rs b/CI-Examples/rust/src/main.rs
@@ -11,7 +11,7 @@ async fn hello_world(_req: Request<Body>) -> Result<Response<Body>, Infallible>
 // because you need to specify in the Gramine manifest the maximal number of threads per
 // process, and ideally this wouldn't depend on your hardware.
 //
-// See sgx.thread_num in the manifest.
+// See sgx.max_threads in the manifest.
 #[tokio::main(worker_threads = 4)]
 async fn main() {
     let addr = SocketAddr::from(([127, 0, 0, 1], 3000));

diff --git a/CI-Examples/sqlite/manifest.template b/CI-Examples/sqlite/manifest.template
@@ -28,7 +28,7 @@ fs.insecure__keys.default = "ffeeddccbbaa99887766554433221100"
 
 sgx.debug = true
 sgx.enclave_size = "256M"
-sgx.thread_num = 4
+sgx.max_threads = 4
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/Documentation/manifest-syntax.rst b/Documentation/manifest-syntax.rst
@@ -496,7 +496,7 @@ Number of threads
 
 ::
 
-    sgx.thread_num = [NUM]
+    sgx.max_threads = [NUM]
     (Default: 4)
 
 This syntax specifies the maximum number of threads that can be created inside
@@ -518,9 +518,9 @@ Note that Gramine uses several helper threads internally:
   each time a new pipe is created. It terminates itself immediately after the
   TLS handshake is performed.
 
-Given these internal threads, ``sgx.thread_num`` should be set to at least ``4``
-even for single-threaded applications (to accommodate for the main thread, the
-IPC thread, the Async thread and one TLS-handshake thread).
+Given these internal threads, ``sgx.max_threads`` should be set to at least
+``4`` even for single-threaded applications (to accommodate for the main thread,
+the IPC thread, the Async thread and one TLS-handshake thread).
 
 
 Number of RPC threads (Exitless feature)
@@ -983,3 +983,12 @@ This syntax specified how much additional memory Gramine used to reserve for its
 internal use (e.g., metadata for trusted files, internal handles,
 etc.). Currently Gramine correctly tracks all internal memory allocations and
 does not require this workaround.
+
+Number of threads (deprecated syntax)
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+::
+
+    sgx.thread_num = [NUM]
+
+This name was ambiguous and was replaced with ``sgx.max_threads``.
diff --git a/Documentation/performance.rst b/Documentation/performance.rst
@@ -181,22 +181,22 @@ for each system call and exit the enclave. The feature can be disabled by
 specifying ``sgx.insecure__rpc_thread_num = 0``.
 
 You must decide how many untrusted helper RPC threads your application needs. A
-rule of thumb: specify ``sgx.insecure__rpc_thread_num == sgx.thread_num``, i.e.,
-the number of untrusted RPC threads should be the same as the number of enclave
-threads. For example, native Redis 6.0 uses 3-4 enclave threads during its
-execution, plus Gramine uses another 1-2 helper enclave threads. So Redis
-manifest has an over-approximation of this number: ``sgx.thread_num = 8``. Thus,
+rule of thumb: specify ``sgx.insecure__rpc_thread_num == sgx.max_threads``,
+i.e., the number of untrusted RPC threads should be the same as the number of
+enclave threads. For example, native Redis 6.0 uses 3-4 enclave threads during
+its execution, plus Gramine uses another 1-2 helper enclave threads. So Redis
+manifest has an over-approximation of this number: ``sgx.max_threads = 8``. Thus,
 to correctly enable the Exitless feature, specify
 ``sgx.insecure__rpc_thread_num = 8``. Here is an example:
 
 ::
 
-   # exitless disabled: `sgx.thread_num = 8` and `sgx.insecure__rpc_thread_num = 0`
+   # exitless disabled: `sgx.max_threads = 8` and `sgx.insecure__rpc_thread_num = 0`
    CI-Examples/redis$ gramine-sgx redis-server --save '' --protected-mode no &
    CI-Examples/redis$ src/src/redis-benchmark -t set
    43010.75 requests per second
 
-   # exitless enabled: `sgx.thread_num = 8` and `sgx.insecure__rpc_thread_num = 8`
+   # exitless enabled: `sgx.max_threads = 8` and `sgx.insecure__rpc_thread_num = 8`
    CI-Examples/redis$ gramine-sgx redis-server --save '' --protected-mode no &
    CI-Examples/redis$ src/src/redis-benchmark -t set
    68119.89 requests per second
@@ -419,8 +419,8 @@ enclave size by tweaking ``sgx.enclave_size = "512M"``,
 doesn't help, it could be due to insufficient stack size: in this case try to
 increase ``sys.stack.size = "256K"``, ``sys.stack.size = "2M"``,
 ``sys.stack.size = "4M"`` and so on. Finally, if Gramine complains about
-insufficient number of TCSs or threads, increase ``sgx.thread_num = 4``,
-``sgx.thread_num = 8``, ``sgx.thread_num = 16``, and so on.
+insufficient number of TCSs or threads, increase ``sgx.max_threads = 4``,
+``sgx.max_threads = 8``, ``sgx.max_threads = 16``, and so on.
 
 Do not forget about the cost of software encryption! Gramine transparently
 encrypts many means of communication:

diff --git a/common/include/toml_utils.h b/common/include/toml_utils.h
@@ -32,7 +32,7 @@ int toml_bool_in(const toml_table_t* root, const char* key, bool defaultval, boo
  * \brief Find an integer key-value in TOML manifest.
  *
  * \param root        Root table of the TOML manifest.
- * \param key         Dotted key (e.g. "sgx.thread_num").
+ * \param key         Dotted key (e.g. "sgx.max_threads").
  * \param defaultval  `retval` is set to this value if not found in the manifest.
  * \param retval      Pointer to output integer.
  *

diff --git a/libos/test/abi/x86_64/manifest.template b/libos/test/abi/x86_64/manifest.template
@@ -8,7 +8,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
-sgx.thread_num = 4
+sgx.max_threads = 4
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/libos/test/abi/x86_64/stack_arg.manifest.template b/libos/test/abi/x86_64/stack_arg.manifest.template
@@ -13,7 +13,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
-sgx.thread_num = 4
+sgx.max_threads = 4
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/libos/test/abi/x86_64/stack_env.manifest.template b/libos/test/abi/x86_64/stack_env.manifest.template
@@ -13,7 +13,7 @@ fs.mounts = [
 
 sgx.nonpie_binary = true
 sgx.debug = true
-sgx.thread_num = 4
+sgx.max_threads = 4
 
 sgx.trusted_files = [
   "file:{{ gramine.libos }}",

diff --git a/libos/test/fs/manifest.template b/libos/test/fs/manifest.template
@@ -22,7 +22,7 @@ fs.insecure__keys.default = "ffeeddccbbaa99887766554433221100"
 
 sgx.nonpie_binary = true
 sgx.debug = true
-sgx.thread_num = 16
+sgx.max_threads = 16
 
 sgx.allowed_files = [
   "file:tmp/",

diff --git a/libos/test/regression/bootstrap_cpp.manifest.template b/libos/test/regression/bootstrap_cpp.manifest.template
@@ -13,7 +13,7 @@ fs.mounts = [
   { path = "/usr/{{ arch_libdir }}", uri = "file:/usr/{{ arch_libdir }}" },
 ]
 
-sgx.thread_num = 8
+sgx.max_threads = 8
 sgx.nonpie_binary = true
 sgx.debug = true
 

diff --git a/libos/test/regression/manifest.template b/libos/test/regression/manifest.template
@@ -21,7 +21,7 @@ fs.mounts = [
   { type = "encrypted", path = "/encrypted_file_mrsigner.dat", uri = "file:encrypted_file_mrsigner.dat", key_name = "_sgx_mrsigner" },
 ]
 
-sgx.thread_num = 16
+sgx.max_threads = 16
 sgx.nonpie_binary = true
 sgx.debug = true
 

diff --git a/libos/test/regression/multi_pthread.manifest.template b/libos/test/regression/multi_pthread.manifest.template
@@ -9,7 +9,7 @@ fs.mounts = [
 ]
 
 # app runs with 4 parallel threads + Gramine has couple internal threads
-sgx.thread_num = 8
+sgx.max_threads = 8
 
 sgx.nonpie_binary = true
 sgx.debug = true

diff --git a/libos/test/regression/multi_pthread_exitless.manifest.template b/libos/test/regression/multi_pthread_exitless.manifest.template
@@ -9,6 +9,7 @@ fs.mounts = [
 ]
 
 # app runs with 4 parallel threads + Gramine has couple internal threads
+# TODO: legacy `sgx.thread_num` name just for testing, deprecated in v1.4, remove in v1.5
 sgx.thread_num = 8
 sgx.insecure__rpc_thread_num = 8
 

diff --git a/libos/test/regression/openmp.manifest.template b/libos/test/regression/openmp.manifest.template
@@ -21,7 +21,7 @@ fs.mounts = [
   { path = "/usr/{{ arch_libdir }}", uri = "file:/usr/{{ arch_libdir }}" },
 ]
 
-sgx.thread_num = 32
+sgx.max_threads = 32
 sgx.nonpie_binary = true
 sgx.debug = true
 

diff --git a/libos/test/regression/pthread_set_get_affinity.c b/libos/test/regression/pthread_set_get_affinity.c
@@ -19,7 +19,7 @@
 
 #define MAIN_THREAD_CNT         1
 #define INTERNAL_THREAD_CNT     2
-#define MANIFEST_SGX_THREAD_CNT 8 /* corresponds to sgx.thread_num in the manifest template */
+#define MANIFEST_SGX_THREAD_CNT 8 /* corresponds to sgx.max_threads in the manifest template */
 
 /* barrier to synchronize between parent and children */
 pthread_barrier_t barrier;
@@ -88,7 +88,7 @@ int main(int argc, const char** argv) {
         errx(EXIT_FAILURE, "Parent should have affinity set to all online cores!");
     }
 
-    /* If you want to run on all cores then increase sgx.thread_num in the manifest.template and
+    /* If you want to run on all cores then increase sgx.max_threads in the manifest.template and
      * also set MANIFEST_SGX_THREAD_CNT to the same value.
      */
     size_t numthreads = MIN(online_cores, (MANIFEST_SGX_THREAD_CNT

diff --git a/libos/test/regression/shebang_test_script.manifest.template b/libos/test/regression/shebang_test_script.manifest.template
@@ -10,7 +10,7 @@ fs.mounts = [
   { path = "/bin", uri = "file:/bin" },
 ]
 
-sgx.thread_num = 16
+sgx.max_threads = 16
 sgx.nonpie_binary = true
 sgx.debug = true
 

diff --git a/pal/regression/Thread2.manifest.template b/pal/regression/Thread2.manifest.template
@@ -1,6 +1,6 @@
 loader.entrypoint = "file:{{ binary_dir }}/{{ entrypoint }}"
 
-sgx.thread_num = 2
+sgx.max_threads = 2
 sgx.enable_stats = true
 sgx.nonpie_binary = true
 sgx.debug = true

diff --git a/pal/regression/Thread2_exitless.manifest.template b/pal/regression/Thread2_exitless.manifest.template
@@ -2,7 +2,7 @@
 
 loader.entrypoint = "file:{{ binary_dir }}/{{ entrypoint }}"
 
-sgx.thread_num = 2
+sgx.max_threads = 2
 sgx.insecure__rpc_thread_num = 2
 sgx.enable_stats = true
 sgx.nonpie_binary = true

diff --git a/pal/src/host/linux-sgx/host_main.c b/pal/src/host/linux-sgx/host_main.c
@@ -663,26 +663,41 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info,
     }
 
     int64_t thread_num_int64;
-    ret = toml_int_in(manifest_root, "sgx.thread_num", /*defaultval=*/0, &thread_num_int64);
+    ret = toml_int_in(manifest_root, "sgx.max_threads", /*defaultval=*/-1, &thread_num_int64);
     if (ret < 0) {
-        log_error("Cannot parse 'sgx.thread_num'");
+        log_error("Cannot parse 'sgx.max_threads'");
         ret = -EINVAL;
         goto out;
     }
 
     if (thread_num_int64 < 0) {
-        log_error("Negative 'sgx.thread_num' is impossible");
+        /* TODO: sgx.thread_num is deprecated in v1.4, remove in v1.5 */
+        ret = toml_int_in(manifest_root, "sgx.thread_num", /*defaultval=*/-1, &thread_num_int64);
+        if (ret < 0) {
+            log_error("Cannot parse 'sgx.thread_num'");
+            ret = -EINVAL;
+            goto out;
+        }
+        if (thread_num_int64 < 0) {
+            log_error("'sgx.max_threads' not found in the manifest");
+            ret = -EINVAL;
+            goto out;
+        }
+        log_error("Detected deprecated syntax: 'sgx.thread_num'. Consider switching to "
+                  "'sgx.max_threads'.");
+    }
+
+    if (!thread_num_int64) {
+        log_error("'sgx.max_threads' must be a positive number");
         ret = -EINVAL;
         goto out;
     }
-
-    enclave_info->thread_num = thread_num_int64 ?: 1;
-
-    if (enclave_info->thread_num > MAX_DBG_THREADS) {
-        log_error("Too large 'sgx.thread_num', maximum allowed is %d", MAX_DBG_THREADS);
+    if (thread_num_int64 > MAX_DBG_THREADS) {
+        log_error("Too large 'sgx.max_threads', maximum allowed is %d", MAX_DBG_THREADS);
         ret = -EINVAL;
         goto out;
     }
+    enclave_info->thread_num = thread_num_int64;
 
     int64_t rpc_thread_num_int64;
     ret = toml_int_in(manifest_root, "sgx.insecure__rpc_thread_num", /*defaultval=*/0,
@@ -699,14 +714,13 @@ static int parse_loader_config(char* manifest, struct pal_enclave* enclave_info,
         goto out;
     }
 
-    enclave_info->rpc_thread_num = rpc_thread_num_int64;
-
-    if (enclave_info->rpc_thread_num > MAX_RPC_THREADS) {
+    if (rpc_thread_num_int64 > MAX_RPC_THREADS) {
         log_error("Too large 'sgx.insecure__rpc_thread_num', maximum allowed is %d",
                   MAX_RPC_THREADS);
         ret = -EINVAL;
         goto out;
     }
+    enclave_info->rpc_thread_num = rpc_thread_num_int64;
 
     if (enclave_info->rpc_thread_num && enclave_info->thread_num > RPC_QUEUE_SIZE) {
         log_error("Too many threads for exitless feature (more than capacity of RPC queue)");

diff --git a/pal/src/host/linux-sgx/host_thread.c b/pal/src/host/linux-sgx/host_thread.c
@@ -176,7 +176,7 @@ int pal_thread_init(void* tcbptr) {
     if (!tcb->tcs) {
         log_error(
             "There are no available TCS pages left for a new thread!\n"
-            "Please try to increase sgx.thread_num in the manifest.\n"
+            "Please try to increase sgx.max_threads in the manifest.\n"
             "The current value is %d",
             g_enclave_thread_num);
         ret = -ENOMEM;

diff --git a/python/graminelibos/manifest.py b/python/graminelibos/manifest.py
@@ -87,7 +87,11 @@ def __init__(self, manifest_str):
         sgx = manifest.setdefault('sgx', {})
         sgx.setdefault('trusted_files', [])
         sgx.setdefault('enclave_size', DEFAULT_ENCLAVE_SIZE)
-        sgx.setdefault('thread_num', DEFAULT_THREAD_NUM)
+
+        # TODO: sgx.thread_num is deprecated in v1.4, simplify below logic in v1.5
+        if 'thread_num' not in sgx:
+            sgx.setdefault('max_threads', DEFAULT_THREAD_NUM)
+
         sgx.setdefault('isvprodid', 0)
         sgx.setdefault('isvsvn', 0)
         sgx.setdefault('remote_attestation', "none")