[Code] disk watermark supports percentage and absolute modes (#42987)

* [Code] disk watermark supports percentage and absolute modes

* add unit tests

* use percentage mode by default

x-pack/legacy/plugins/code/index.ts

@@ -14,6 +14,7 @@ import { CoreSetup, PluginInitializerContext } from 'src/core/server';
 import { APP_TITLE } from './common/constants';
 import { LanguageServers, LanguageServersDeveloping } from './server/lsp/language_servers';
 import { codePlugin } from './server';
+import { DEFAULT_WATERMARK_LOW_PERCENTAGE } from './server/disk_watermark';
 
 export type RequestFacade = Legacy.Request;
 export type RequestQueryFacade = RequestQuery;
@@ -104,7 +105,7 @@ export const code = (kibana: any) =>
         }).default(),
         disk: Joi.object({
           thresholdEnabled: Joi.bool().default(true),
-          watermarkLowMb: Joi.number().default(2048),
+          watermarkLow: Joi.string().default(`${DEFAULT_WATERMARK_LOW_PERCENTAGE}%`),
         }).default(),
         maxWorkspace: Joi.number().default(5), // max workspace folder for each language server
         enableGlobalReference: Joi.boolean().default(false), // Global reference as optional feature for now

x-pack/legacy/plugins/code/server/disk_watermark.test.ts

@@ -0,0 +1,193 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License;
+ * you may not use this file except in compliance with the Elastic License.
+ */
+
+import sinon from 'sinon';
+
+import { DiskWatermarkService } from './disk_watermark';
+import { Logger } from './log';
+import { ServerOptions } from './server_options';
+import { ConsoleLoggerFactory } from './utils/console_logger_factory';
+
+const log: Logger = new ConsoleLoggerFactory().getLogger(['test']);
+
+afterEach(() => {
+  sinon.restore();
+});
+
+test('Disk watermark check in percentage mode', async () => {
+  const diskCheckerStub = sinon.stub();
+  diskCheckerStub
+    .onFirstCall()
+    .resolves({
+      free: 5,
+      size: 10,
+    })
+    .onSecondCall()
+    .returns({
+      free: 1,
+      size: 10,
+    });
+  const diskWatermarkService = new DiskWatermarkService(
+    diskCheckerStub,
+    {
+      disk: {
+        thresholdEnabled: true,
+        watermarkLow: '80%',
+      },
+      repoPath: '/',
+    } as ServerOptions,
+    log
+  );
+
+  expect(await diskWatermarkService.isLowWatermark()).toBeFalsy(); // 50% usage
+  expect(await diskWatermarkService.isLowWatermark()).toBeTruthy(); // 90% usage
+});
+
+test('Disk watermark check in absolute mode in kb', async () => {
+  const diskCheckerStub = sinon.stub();
+  diskCheckerStub
+    .onFirstCall()
+    .resolves({
+      free: 8 * Math.pow(1024, 1),
+      size: 10000,
+    })
+    .onSecondCall()
+    .returns({
+      free: 2 * Math.pow(1024, 1),
+      size: 10000,
+    });
+  const diskWatermarkService = new DiskWatermarkService(
+    diskCheckerStub,
+    {
+      disk: {
+        thresholdEnabled: true,
+        watermarkLow: '4kb',
+      },
+      repoPath: '/',
+    } as ServerOptions,
+    log
+  );
+
+  expect(await diskWatermarkService.isLowWatermark()).toBeFalsy(); // 8kb available
+  expect(await diskWatermarkService.isLowWatermark()).toBeTruthy(); // 2kb available
+});
+
+test('Disk watermark check in absolute mode in mb', async () => {
+  const diskCheckerStub = sinon.stub();
+  diskCheckerStub
+    .onFirstCall()
+    .resolves({
+      free: 8 * Math.pow(1024, 2),
+      size: 10000,
+    })
+    .onSecondCall()
+    .returns({
+      free: 2 * Math.pow(1024, 2),
+      size: 10000,
+    });
+  const diskWatermarkService = new DiskWatermarkService(
+    diskCheckerStub,
+    {
+      disk: {
+        thresholdEnabled: true,
+        watermarkLow: '4mb',
+      },
+      repoPath: '/',
+    } as ServerOptions,
+    log
+  );
+
+  expect(await diskWatermarkService.isLowWatermark()).toBeFalsy(); // 8mb available
+  expect(await diskWatermarkService.isLowWatermark()).toBeTruthy(); // 2mb available
+});
+
+test('Disk watermark check in absolute mode in gb', async () => {
+  const diskCheckerStub = sinon.stub();
+  diskCheckerStub
+    .onFirstCall()
+    .resolves({
+      free: 8 * Math.pow(1024, 3),
+      size: 10000,
+    })
+    .onSecondCall()
+    .returns({
+      free: 2 * Math.pow(1024, 3),
+      size: 10000,
+    });
+  const diskWatermarkService = new DiskWatermarkService(
+    diskCheckerStub,
+    {
+      disk: {
+        thresholdEnabled: true,
+        watermarkLow: '4gb',
+      },
+      repoPath: '/',
+    } as ServerOptions,
+    log
+  );
+
+  expect(await diskWatermarkService.isLowWatermark()).toBeFalsy(); // 8gb available
+  expect(await diskWatermarkService.isLowWatermark()).toBeTruthy(); // 2gb available
+});
+
+test('Disk watermark check in absolute mode in tb', async () => {
+  const diskCheckerStub = sinon.stub();
+  diskCheckerStub
+    .onFirstCall()
+    .resolves({
+      free: 8 * Math.pow(1024, 4),
+      size: 10000,
+    })
+    .onSecondCall()
+    .returns({
+      free: 2 * Math.pow(1024, 4),
+      size: 10000,
+    });
+  const diskWatermarkService = new DiskWatermarkService(
+    diskCheckerStub,
+    {
+      disk: {
+        thresholdEnabled: true,
+        watermarkLow: '4tb',
+      },
+      repoPath: '/',
+    } as ServerOptions,
+    log
+  );
+
+  expect(await diskWatermarkService.isLowWatermark()).toBeFalsy(); // 8tb available
+  expect(await diskWatermarkService.isLowWatermark()).toBeTruthy(); // 2tb available
+});
+
+test('Disk watermark check in invalid config', async () => {
+  const diskCheckerStub = sinon.stub();
+  diskCheckerStub
+    .onFirstCall()
+    .resolves({
+      free: 50,
+      size: 100,
+    })
+    .onSecondCall()
+    .returns({
+      free: 5,
+      size: 100,
+    });
+  const diskWatermarkService = new DiskWatermarkService(
+    diskCheckerStub,
+    {
+      disk: {
+        thresholdEnabled: true,
+        // invalid config, will fallback with 90% by default
+        watermarkLow: '1234',
+      },
+      repoPath: '/',
+    } as ServerOptions,
+    log
+  );
+
+  expect(await diskWatermarkService.isLowWatermark()).toBeFalsy(); // 50% usage
+  expect(await diskWatermarkService.isLowWatermark()).toBeTruthy(); // 95% usage
+});

x-pack/legacy/plugins/code/server/disk_watermark.ts

@@ -3,19 +3,104 @@
  * or more contributor license agreements. Licensed under the Elastic License;
  * you may not use this file except in compliance with the Elastic License.
  */
+import { i18n } from '@kbn/i18n';
+import { CheckDiskSpaceResult } from 'check-disk-space';
 
-import checkDiskSpace from 'check-disk-space';
+import { Logger } from './log';
+import { ServerOptions } from './server_options';
+
+export const DEFAULT_WATERMARK_LOW_PERCENTAGE = 90;
+
+export type DiskCheckResult = CheckDiskSpaceResult;
+export type DiskSpaceChecker = (path: string) => Promise<DiskCheckResult>;
 
 export class DiskWatermarkService {
-  constructor(private readonly diskWatermarkLowMb: number, private readonly repoPath: string) {}
+  // True for percentage mode (e.g. 90%), false for absolute mode (e.g. 500mb)
+  private percentageMode: boolean = true;
+  private watermark: number = DEFAULT_WATERMARK_LOW_PERCENTAGE;
+  private enabled: boolean = false;
+
+  constructor(
+    private readonly diskSpaceChecker: DiskSpaceChecker,
+    private readonly serverOptions: ServerOptions,
+    private readonly logger: Logger
+  ) {
+    this.enabled = this.serverOptions.disk.thresholdEnabled;
+    if (this.enabled) {
+      this.parseWatermarkConfigString(this.serverOptions.disk.watermarkLow);
+    }
+  }
 
   public async isLowWatermark(): Promise<boolean> {
+    if (!this.enabled) {
+      return false;
+    }
+
     try {
-      const { free } = await checkDiskSpace(this.repoPath);
-      const availableMb = free / 1024 / 1024;
-      return availableMb <= this.diskWatermarkLowMb;
+      const res = await this.diskSpaceChecker(this.serverOptions.repoPath);
+      const { free, size } = res;
+      if (this.percentageMode) {
+        const percentage = ((size - free) * 100) / size;
+        return percentage > this.watermark;
+      } else {
+        return free <= this.watermark;
+      }
     } catch (err) {
       return true;
     }
   }
+
+  public diskWatermarkViolationMessage(): string {
+    if (this.percentageMode) {
+      return i18n.translate('xpack.code.git.diskWatermarkLowPercentageMessage', {
+        defaultMessage: `Disk usage watermark level higher than {watermark}`,
+        values: {
+          watermark: this.serverOptions.disk.watermarkLow,
+        },
+      });
+    } else {
+      return i18n.translate('xpack.code.git.diskWatermarkLowMessage', {
+        defaultMessage: `Available disk space lower than {watermark}`,
+        values: {
+          watermark: this.serverOptions.disk.watermarkLow,
+        },
+      });
+    }
+  }
+
+  private parseWatermarkConfigString(diskWatermarkLow: string) {
+    // Including undefined, null and empty string.
+    if (!diskWatermarkLow) {
+      this.logger.error(
+        `Empty disk watermark config for Code. Fallback with default value (${DEFAULT_WATERMARK_LOW_PERCENTAGE}%)`
+      );
+      return;
+    }
+
+    try {
+      const str = diskWatermarkLow.trim().toLowerCase();
+      if (str.endsWith('%')) {
+        this.percentageMode = true;
+        this.watermark = parseInt(str.substr(0, str.length - 1), 10);
+      } else if (str.endsWith('kb')) {
+        this.percentageMode = false;
+        this.watermark = parseInt(str.substr(0, str.length - 2), 10) * Math.pow(1024, 1);
+      } else if (str.endsWith('mb')) {
+        this.percentageMode = false;
+        this.watermark = parseInt(str.substr(0, str.length - 2), 10) * Math.pow(1024, 2);
+      } else if (str.endsWith('gb')) {
+        this.percentageMode = false;
+        this.watermark = parseInt(str.substr(0, str.length - 2), 10) * Math.pow(1024, 3);
+      } else if (str.endsWith('tb')) {
+        this.percentageMode = false;
+        this.watermark = parseInt(str.substr(0, str.length - 2), 10) * Math.pow(1024, 4);
+      } else {
+        throw new Error('Unrecognized unit for disk size config.');
+      }
+    } catch (error) {
+      this.logger.error(
+        `Invalid disk watermark config for Code. Fallback with default value (${DEFAULT_WATERMARK_LOW_PERCENTAGE}%)`
+      );
+    }
+  }
 }

x-pack/legacy/plugins/code/server/init_workers.ts

@@ -4,7 +4,9 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
+import checkDiskSpace from 'check-disk-space';
 import { Server } from 'hapi';
+
 import { DiskWatermarkService } from './disk_watermark';
 import { EsClient, Esqueue } from './lib/esqueue';
 import { LspService } from './lsp/lsp_service';
@@ -44,10 +46,7 @@ export function initWorkers(
 
   const repoServiceFactory: RepositoryServiceFactory = new RepositoryServiceFactory();
 
-  const watermarkService = new DiskWatermarkService(
-    serverOptions.disk.watermarkLowMb,
-    serverOptions.repoPath
-  );
+  const watermarkService = new DiskWatermarkService(checkDiskSpace, serverOptions, log);
   const cloneWorker = new CloneWorker(
     queue,
     log,

x-pack/legacy/plugins/code/server/queue/abstract_git_worker.ts

@@ -4,8 +4,6 @@
  * you may not use this file except in compliance with the Elastic License.
  */
 
-import { i18n } from '@kbn/i18n';
-
 import {
   CloneProgress,
   CloneWorkerProgress,
@@ -39,19 +37,10 @@ export abstract class AbstractGitWorker extends AbstractWorker {
   }
 
   public async executeJob(_: Job): Promise<WorkerResult> {
-    const { thresholdEnabled, watermarkLowMb } = this.serverOptions.disk;
-    if (thresholdEnabled) {
-      const isLowWatermark = await this.watermarkService.isLowWatermark();
-      if (isLowWatermark) {
-        const msg = i18n.translate('xpack.code.git.diskWatermarkLowMessage', {
-          defaultMessage: `Disk watermark level lower than {watermarkLowMb} MB`,
-          values: {
-            watermarkLowMb,
-          },
-        });
-        this.log.error(msg);
-        throw new Error(msg);
-      }
+    if (await this.watermarkService.isLowWatermark()) {
+      const msg = this.watermarkService.diskWatermarkViolationMessage();
+      this.log.error(msg);
+      throw new Error(msg);
     }
 
     return new Promise<WorkerResult>((resolve, reject) => {