[core.savedObjects] Fix maxImportExportSize config & update docs. (#9…

…4019)
elastic · Mar 18, 2021 · d1040f0 · d1040f0
1 parent ab0f45c
commit d1040f0
Show file tree

Hide file tree

Showing 17 changed files with 60 additions and 25 deletions.
diff --git a/api_docs/core_saved_objects.json b/api_docs/core_saved_objects.json
@@ -4581,15 +4581,15 @@
                 "description": [],
                 "source": {
                   "path": "src/core/server/saved_objects/export/errors.ts",
-                  "lineNumber": 34
+                  "lineNumber": 36
                 }
               }
             ],
             "tags": [],
             "returnComment": [],
             "source": {
               "path": "src/core/server/saved_objects/export/errors.ts",
-              "lineNumber": 34
+              "lineNumber": 36
             }
           },
           {
@@ -4628,7 +4628,7 @@
                 "description": [],
                 "source": {
                   "path": "src/core/server/saved_objects/export/errors.ts",
-                  "lineNumber": 43
+                  "lineNumber": 45
                 }
               },
               {
@@ -4641,15 +4641,15 @@
                 "description": [],
                 "source": {
                   "path": "src/core/server/saved_objects/export/errors.ts",
-                  "lineNumber": 43
+                  "lineNumber": 45
                 }
               }
             ],
             "tags": [],
             "returnComment": [],
             "source": {
               "path": "src/core/server/saved_objects/export/errors.ts",
-              "lineNumber": 43
+              "lineNumber": 45
             }
           },
           {
@@ -4681,15 +4681,15 @@
                 "description": [],
                 "source": {
                   "path": "src/core/server/saved_objects/export/errors.ts",
-                  "lineNumber": 58
+                  "lineNumber": 60
                 }
               }
             ],
             "tags": [],
             "returnComment": [],
             "source": {
               "path": "src/core/server/saved_objects/export/errors.ts",
-              "lineNumber": 58
+              "lineNumber": 60
             }
           }
         ],

diff --git a/docs/api/saved-objects/export.asciidoc b/docs/api/saved-objects/export.asciidoc
@@ -36,6 +36,9 @@ experimental[] Retrieve sets of saved objects that you want to import into {kib}
 
 TIP: You must include `type` or `objects` in the request body.
 
+NOTE: The <<savedObjects-maxImportExportSize, `savedObjects.maxImportExportSize`>> configuration setting
+limits the number of saved objects which may be exported.
+
 [[saved-objects-api-export-request-response-body]]
 ==== Response body
 

diff --git a/docs/api/saved-objects/import.asciidoc b/docs/api/saved-objects/import.asciidoc
@@ -53,6 +53,11 @@ The request body must include the multipart/form-data type.
 
 `file`::
   A file exported using the export API.
++
+NOTE: The <<savedObjects-maxImportExportSize, `savedObjects.maxImportExportSize`>> configuration setting
+limits the number of saved objects which may be included in this file. Similarly, the
+<<savedObjects-maxImportPayloadBytes, `savedObjects.maxImportPayloadBytes`>> setting limits the overall
+size of the file that can be imported.
 
 [[saved-objects-api-import-response-body]]
 ==== Response body

diff --git a/docs/management/managing-saved-objects.asciidoc b/docs/management/managing-saved-objects.asciidoc
@@ -67,6 +67,12 @@ navigate to the NDJSON file that
 represents the objects to import.  By default,
 saved objects already in {kib} are overwritten.
 
+NOTE: The <<savedObjects-maxImportExportSize, `savedObjects.maxImportExportSize`>> configuration setting
+limits the number of saved objects which may be included in this file. Similarly, the
+<<savedObjects-maxImportPayloadBytes, `savedObjects.maxImportPayloadBytes`>> setting limits the overall
+size of the file that can be imported.
+
+
 [float]
 ==== Export
 
@@ -78,6 +84,10 @@ You have two options for exporting saved objects.
 This action creates an NDJSON with all your saved objects. By default, the NDJSON includes child objects that are related to the saved
 objects. Exported dashboards include their associated index patterns.
 
+NOTE: The <<savedObjects-maxImportExportSize, `savedObjects.maxImportExportSize`>> configuration setting
+limits the number of saved objects which may be exported.
+
+
 [float]
 [role="xpack"]
 [[managing-saved-objects-copy-to-space]]

diff --git a/docs/setup/settings.asciidoc b/docs/setup/settings.asciidoc
@@ -455,6 +455,20 @@ manner that is inconsistent with `/proc/self/cgroup`.
  | Override for cgroup cpuacct path when mounted
 in a manner that is inconsistent with `/proc/self/cgroup`.
 
+|[[savedObjects-maxImportExportSize]] `savedObjects.maxImportExportSize:`
+ | The maximum count of saved objects that can be imported or exported.
+This setting exists to prevent the {kib} server from runnning out of memory when handling
+large numbers of saved objects. It is recommended to only raise this setting if you are
+confident your server can hold this many objects in memory.
+*Default: `10000`*
+
+|[[savedObjects-maxImportPayloadBytes]] `savedObjects.maxImportPayloadBytes:`
+ | The maximum byte size of a saved objects import that the {kib} server will accept.
+This setting exists to prevent the {kib} server from runnning out of memory when handling
+a large import payload. Note that this setting overrides the more general
+<<server-maxPayloadBytes, `server.maxPayloadBytes`>> for saved object imports only.
+*Default: `26214400`*
+
 |[[server-basePath]] `server.basePath:`
  | Enables you to specify a path to mount {kib} at if you are
 running behind a proxy. Use the <<server-rewriteBasePath, `server.rewriteBasePath`>> setting to tell {kib}
@@ -495,7 +509,7 @@ back end server. To allow remote users to connect, set the value to the IP addre
  | The number of milliseconds to wait for additional data before restarting
 the <<server-socketTimeout, `server.socketTimeout`>> counter. *Default: `"120000"`*
 
-| `server.maxPayloadBytes:`
+|[[server-maxPayloadBytes]] `server.maxPayloadBytes:`
  | The maximum payload size in bytes
 for incoming server requests. *Default: `1048576`*
 

diff --git a/src/core/server/core_usage_data/core_usage_data_service.mock.ts b/src/core/server/core_usage_data/core_usage_data_service.mock.ts
@@ -106,7 +106,7 @@ const createStartContractMock = () => {
           },
           savedObjects: {
             customIndex: false,
-            maxImportExportSizeBytes: 10000,
+            maxImportExportSize: 10000,
             maxImportPayloadBytes: 26214400,
           },
         },

diff --git a/src/core/server/core_usage_data/core_usage_data_service.test.ts b/src/core/server/core_usage_data/core_usage_data_service.test.ts
@@ -239,7 +239,7 @@ describe('CoreUsageDataService', () => {
               },
               "savedObjects": Object {
                 "customIndex": false,
-                "maxImportExportSizeBytes": 10000,
+                "maxImportExportSize": 10000,
                 "maxImportPayloadBytes": 26214400,
               },
             },

diff --git a/src/core/server/core_usage_data/core_usage_data_service.ts b/src/core/server/core_usage_data/core_usage_data_service.ts
@@ -235,7 +235,7 @@ export class CoreUsageDataService implements CoreService<CoreUsageDataSetup, Cor
         savedObjects: {
           customIndex: isCustomIndex(this.kibanaConfig!.index),
           maxImportPayloadBytes: this.soConfig.maxImportPayloadBytes.getValueInBytes(),
-          maxImportExportSizeBytes: this.soConfig.maxImportExportSize.getValueInBytes(),
+          maxImportExportSize: this.soConfig.maxImportExportSize,
         },
       },
       environment: {

diff --git a/src/core/server/core_usage_data/types.ts b/src/core/server/core_usage_data/types.ts
@@ -229,7 +229,7 @@ export interface CoreConfigUsageData {
   savedObjects: {
     customIndex: boolean;
     maxImportPayloadBytes: number;
-    maxImportExportSizeBytes: number;
+    maxImportExportSize: number;
   };
 
   // uiSettings: {

diff --git a/src/core/server/saved_objects/export/errors.ts b/src/core/server/saved_objects/export/errors.ts
@@ -27,7 +27,9 @@ export class SavedObjectsExportError extends Error {
   static exportSizeExceeded(limit: number) {
     return new SavedObjectsExportError(
       'export-size-exceeded',
-      `Can't export more than ${limit} objects`
+      `Can't export more than ${limit} objects. ` +
+        'If your server has enough memory, this limit can be increased ' +
+        'by adjusting the "savedObjects.maxImportExportSize" setting.'
     );
   }
 

diff --git a/src/core/server/saved_objects/export/saved_objects_exporter.test.ts b/src/core/server/saved_objects/export/saved_objects_exporter.test.ts
@@ -877,7 +877,9 @@ describe('getSortedObjectsForExport()', () => {
           request,
           types: ['index-pattern', 'search'],
         })
-      ).rejects.toThrowErrorMatchingInlineSnapshot(`"Can't export more than 1 objects"`);
+      ).rejects.toThrowErrorMatchingInlineSnapshot(
+        `"Can't export more than 1 objects. If your server has enough memory, this limit can be increased by adjusting the \\"savedObjects.maxImportExportSize\\" setting."`
+      );
       expect(savedObjectsClient.closePointInTime).toHaveBeenCalledTimes(1);
     });
 
@@ -1112,7 +1114,7 @@ describe('getSortedObjectsForExport()', () => {
         ],
       };
       await expect(exporter.exportByObjects(exportOpts)).rejects.toThrowErrorMatchingInlineSnapshot(
-        `"Can't export more than 1 objects"`
+        `"Can't export more than 1 objects. If your server has enough memory, this limit can be increased by adjusting the \\"savedObjects.maxImportExportSize\\" setting."`
       );
     });
 

diff --git a/src/core/server/saved_objects/saved_objects_config.ts b/src/core/server/saved_objects/saved_objects_config.ts
@@ -28,7 +28,7 @@ export const savedObjectsConfig = {
   path: 'savedObjects',
   schema: schema.object({
     maxImportPayloadBytes: schema.byteSize({ defaultValue: 26214400 }),
-    maxImportExportSize: schema.byteSize({ defaultValue: 10000 }),
+    maxImportExportSize: schema.number({ defaultValue: 10000 }),
   }),
 };
 
@@ -43,7 +43,7 @@ export class SavedObjectConfig {
     rawMigrationConfig: SavedObjectsMigrationConfigType
   ) {
     this.maxImportPayloadBytes = rawConfig.maxImportPayloadBytes.getValueInBytes();
-    this.maxImportExportSize = rawConfig.maxImportExportSize.getValueInBytes();
+    this.maxImportExportSize = rawConfig.maxImportExportSize;
     this.migration = rawMigrationConfig;
   }
 }
diff --git a/src/core/server/saved_objects/saved_objects_service.test.ts b/src/core/server/saved_objects/saved_objects_service.test.ts
@@ -42,7 +42,7 @@ describe('SavedObjectsService', () => {
       }
       return new BehaviorSubject({
         maxImportPayloadBytes: new ByteSizeValue(0),
-        maxImportExportSize: new ByteSizeValue(0),
+        maxImportExportSize: 10000,
       });
     });
     return mockCoreContext.create({ configService, env });

diff --git a/src/core/server/server.api.md b/src/core/server/server.api.md
@@ -452,7 +452,7 @@ export interface CoreConfigUsageData {
     savedObjects: {
         customIndex: boolean;
         maxImportPayloadBytes: number;
-        maxImportExportSizeBytes: number;
+        maxImportExportSize: number;
     };
 }
 

diff --git a/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts b/src/plugins/kibana_usage_collection/server/collectors/core/core_usage_collector.ts
@@ -269,11 +269,10 @@ export function getCoreUsageCollector(
                 'Maximum size of the payload in bytes of saved objects that can be imported.',
             },
           },
-          maxImportExportSizeBytes: {
+          maxImportExportSize: {
             type: 'long',
             _meta: {
-              description:
-                'Maximum size in bytes of saved object that can be imported or exported.',
+              description: 'Maximum count of saved objects that can be imported or exported.',
             },
           },
         },

diff --git a/src/plugins/telemetry/schema/oss_plugins.json b/src/plugins/telemetry/schema/oss_plugins.json
@@ -6675,10 +6675,10 @@
                     "description": "Maximum size of the payload in bytes of saved objects that can be imported."
                   }
                 },
-                "maxImportExportSizeBytes": {
+                "maxImportExportSize": {
                   "type": "long",
                   "_meta": {
-                    "description": "Maximum size in bytes of saved object that can be imported or exported."
+                    "description": "Maximum count of saved objects that can be imported or exported."
                   }
                 }
               }

diff --git a/test/api_integration/apis/saved_objects/export.ts b/test/api_integration/apis/saved_objects/export.ts
@@ -553,7 +553,7 @@ export default function ({ getService }: FtrProviderContext) {
               expect(resp.body).to.eql({
                 statusCode: 400,
                 error: 'Bad Request',
-                message: `Can't export more than 10001 objects`,
+                message: `Can't export more than 10001 objects. If your server has enough memory, this limit can be increased by adjusting the \"savedObjects.maxImportExportSize\" setting.`,
               });
             });
           await supertest