incorporate some better defaults into table-manager configs (#1976)

* Set less aggressive poll interval and a larger grace period for creating new periodic tables in the ksonnet config. Add some rate limiting and retry settings for the table manager bigtable arguments. Signed-off-by: Ed Welch <edward.welch@grafana.com> * Removing the dynamodb provisioning of 0 from config files Signed-off-by: Ed Welch <edward.welch@grafana.com>
grafana · Apr 23, 2020 · e90e3b7 · e90e3b7
1 parent b89a1f8
commit e90e3b7
Show file tree

Hide file tree

Showing 4 changed files with 11 additions and 32 deletions.
diff --git a/cmd/loki/loki-docker-config.yaml b/cmd/loki/loki-docker-config.yaml
@@ -41,15 +41,5 @@ chunk_store_config:
   max_look_back_period: 0s
 
 table_manager:
-  chunk_tables_provisioning:
-    inactive_read_throughput: 0
-    inactive_write_throughput: 0
-    provisioned_read_throughput: 0
-    provisioned_write_throughput: 0
-  index_tables_provisioning:
-    inactive_read_throughput: 0
-    inactive_write_throughput: 0
-    provisioned_read_throughput: 0
-    provisioned_write_throughput: 0
   retention_deletes_enabled: false
   retention_period: 0s
diff --git a/cmd/loki/loki-local-config.yaml b/cmd/loki/loki-local-config.yaml
@@ -41,15 +41,5 @@ chunk_store_config:
   max_look_back_period: 0s
 
 table_manager:
-  chunk_tables_provisioning:
-    inactive_read_throughput: 0
-    inactive_write_throughput: 0
-    provisioned_read_throughput: 0
-    provisioned_write_throughput: 0
-  index_tables_provisioning:
-    inactive_read_throughput: 0
-    inactive_write_throughput: 0
-    provisioned_read_throughput: 0
-    provisioned_write_throughput: 0
   retention_deletes_enabled: false
   retention_period: 0s
diff --git a/production/ksonnet/loki/config.libsonnet b/production/ksonnet/loki/config.libsonnet
@@ -255,18 +255,8 @@
       table_manager: {
         retention_period: 0,
         retention_deletes_enabled: false,
-        index_tables_provisioning: {
-          inactive_read_throughput: 0,
-          inactive_write_throughput: 0,
-          provisioned_read_throughput: 0,
-          provisioned_write_throughput: 0,
-        },
-        chunk_tables_provisioning: {
-          inactive_read_throughput: 0,
-          inactive_write_throughput: 0,
-          provisioned_read_throughput: 0,
-          provisioned_write_throughput: 0,
-        },
+        poll_interval: '10m',
+        creation_grace_period: '3h',
       },
 
       distributor: {

diff --git a/production/ksonnet/loki/table-manager.libsonnet b/production/ksonnet/loki/table-manager.libsonnet
@@ -4,6 +4,15 @@
   table_manager_args::
     $._config.commonArgs {
       target: 'table-manager',
+
+      // Rate limit Bigtable Admin calls.  Google seem to limit to ~100QPS,
+      // and given 2yrs worth of tables (~100) a sync will table 20s.  This
+      // allows you to run upto 20 independant Cortex clusters on the same
+      // Google project before running into issues.
+      'bigtable.grpc-client-rate-limit': 5.0,
+      'bigtable.grpc-client-rate-limit-burst': 5,
+      'bigtable.backoff-on-ratelimits': true,
+      'bigtable.table-cache.enabled': true,
     },
 
   table_manager_container::