[FAI-13685] Write config/catalog files (#97)

faros-ai · Nov 20, 2024 · 625b0bc · 625b0bc
1 parent 49333af
commit 625b0bc
Show file tree

Hide file tree

Showing 10 changed files with 192 additions and 59 deletions.
diff --git a/airbyte-local-cli-nodejs/.gitignore b/airbyte-local-cli-nodejs/.gitignore
@@ -12,4 +12,4 @@ target/
 out/pkg
 sample_command.sh
 *airbyte-local
-test/exec/resources/
+test/exec/resources
diff --git a/airbyte-local-cli-nodejs/src/command.ts b/airbyte-local-cli-nodejs/src/command.ts
@@ -126,7 +126,7 @@ function parseSrcAndDstConfig(argv: string[]) {
 }
 
 // Convert the options to CliOptions type
-function convertToCliOptions(options: any) {
+function convertToCliOptions(options: any): CliOptions {
   return {
     ...options,
     srcImage: options.src,
@@ -135,7 +135,7 @@ function convertToCliOptions(options: any) {
 }
 
 // Validate the input options
-function validateConfigFileInput(config: FarosConfig, inputType: AirbyteConfigInputType) {
+function validateConfigFileInput(config: FarosConfig, inputType: AirbyteConfigInputType): void {
   if (!config.src?.image && !config.srcInputFile) {
     if (inputType === AirbyteConfigInputType.OPTION) {
       throw new Error(`Missing source image. Please use '--src <image>' to provide the source image`);
@@ -156,7 +156,7 @@ function validateConfigFileInput(config: FarosConfig, inputType: AirbyteConfigIn
 }
 
 // parse the command line arguments
-export function parseAndValidateInputs(argv: string[]) {
+export function parseAndValidateInputs(argv: string[]): FarosConfig {
   // Parse the command line arguments
   const program = command().parse(argv);
 
@@ -197,6 +197,7 @@ export function parseAndValidateInputs(argv: string[]) {
     rawMessages: cliOptions.rawMessages ?? false,
     keepContainers: cliOptions.keepContainers ?? false,
     logLevel: cliOptions.logLevel ?? 'info',
+    debug: cliOptions.debug ?? false,
   };
 
   if (cliOptions.srcImage || cliOptions.dstImage) {

diff --git a/airbyte-local-cli-nodejs/src/index.ts b/airbyte-local-cli-nodejs/src/index.ts
@@ -1,14 +1,15 @@
 import {parseAndValidateInputs} from './command';
 import {AirbyteCliContext} from './types';
-import {checkDockerInstalled, cleanUp, createTmpDir, loadStateFile, logger} from './utils';
+import {checkDockerInstalled, cleanUp, createTmpDir, loadStateFile, logger, writeConfig} from './utils';
 
-function main() {
+function main(): void {
   const context: AirbyteCliContext = {};
   try {
     const cfg = parseAndValidateInputs(process.argv);
     checkDockerInstalled();
     context.tmpDir = createTmpDir();
     loadStateFile(context.tmpDir, cfg?.stateFile, cfg?.connectionName);
+    writeConfig(context.tmpDir, cfg);
   } catch (error: any) {
     logger.error(error.message, 'Error');
     cleanUp(context);

diff --git a/airbyte-local-cli-nodejs/src/types.ts b/airbyte-local-cli-nodejs/src/types.ts
@@ -63,4 +63,5 @@ export interface FarosConfig {
   rawMessages: boolean;
   keepContainers: boolean;
   logLevel: string;
+  debug: boolean;
 }
diff --git a/airbyte-local-cli-nodejs/src/utils.ts b/airbyte-local-cli-nodejs/src/utils.ts
@@ -6,7 +6,10 @@ import {sep} from 'node:path';
 import pino from 'pino';
 import pretty from 'pino-pretty';
 
-import {AirbyteCliContext, AirbyteConfig} from './types';
+import {AirbyteCliContext, AirbyteConfig, FarosConfig} from './types';
+
+// constants
+export const FILENAME_PREFIX = 'faros_airbyte_cli';
 
 // Create a pino logger instance
 export const logger = pino(pretty({colorize: true}));
@@ -106,3 +109,51 @@ export function cleanUp(context: AirbyteCliContext): void {
   }
   logger.info('Clean up completed.');
 }
+
+// Write Airbyte config and catalog to temporary dir and a json file
+export function writeConfig(tmpDir: string, config: FarosConfig): void {
+  const airbyteConfig = {
+    src: config.src ?? ({} as AirbyteConfig),
+    dst: config.dst ?? ({} as AirbyteConfig),
+  };
+
+  // write Airbyte config for user's reference
+  // TODO: @FAI-14122 React secrets
+  logger.debug(`Writing Airbyte config for user reference...`);
+  writeFileSync(`${FILENAME_PREFIX}_config.json`, JSON.stringify(airbyteConfig, null, 2));
+  logger.debug(airbyteConfig, `Airbyte config: `);
+  logger.debug(`Airbyte config written to: ${FILENAME_PREFIX}_config.json`);
+
+  // add config `feed_cfg.debug` if debug is enabled
+  const regex = /^farosai\/airbyte-faros-feeds-source.*/;
+  if (config.debug && regex.exec(airbyteConfig.src.image ?? '')) {
+    airbyteConfig.src.config = {
+      ...airbyteConfig.src.config,
+      feed_cfg: {debug: true},
+    };
+  }
+
+  // write config to temporary directory config files
+  logger.debug(`Writing Airbyte config to files...`);
+  const srcConfigFilePath = `${tmpDir}${sep}${FILENAME_PREFIX}_src_config.json`;
+  const dstConfigFilePath = `${tmpDir}${sep}${FILENAME_PREFIX}_dst_config.json`;
+  writeFileSync(srcConfigFilePath, JSON.stringify(airbyteConfig.src.config ?? {}, null, 2));
+  writeFileSync(dstConfigFilePath, JSON.stringify(airbyteConfig.dst.config ?? {}, null, 2));
+  logger.debug(`Airbyte config files written to: ${srcConfigFilePath}, ${dstConfigFilePath}`);
+
+  // write catalog to temporary directory catalog files
+  // TODO: @FAI-14134 Discover catalog
+  logger.debug(`Writing Airbyte catalog to files...`);
+  const srcCatalogFilePath = `${tmpDir}${sep}${FILENAME_PREFIX}_src_catalog.json`;
+  const dstCatalogFilePath = `${tmpDir}${sep}${FILENAME_PREFIX}_dst_catalog.json`;
+  if (
+    (!airbyteConfig.dst.catalog || Object.keys(airbyteConfig.dst.catalog).length === 0) &&
+    airbyteConfig.src.catalog &&
+    Object.keys(airbyteConfig.src.catalog).length > 0
+  ) {
+    airbyteConfig.dst.catalog = airbyteConfig.src.catalog;
+  }
+  writeFileSync(srcCatalogFilePath, JSON.stringify(airbyteConfig.src.catalog ?? {}, null, 2));
+  writeFileSync(dstCatalogFilePath, JSON.stringify(airbyteConfig.dst.catalog ?? {}, null, 2));
+  logger.debug(`Airbyte catalog files written to: ${srcCatalogFilePath}, ${dstCatalogFilePath}`);
+}
diff --git a/airbyte-local-cli-nodejs/test/__snapshots__/utils.it.test.ts.snap b/airbyte-local-cli-nodejs/test/__snapshots__/utils.it.test.ts.snap
@@ -1,10 +1,5 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`loadStateFile should pass with existing state file 1`] = `
-"{"format":"base64/gzip","data":"dGVzdA=="}
-"
-`;
-
 exports[`parseConfigFile should pass 1`] = `
 {
   "dst": {
@@ -32,3 +27,8 @@ exports[`parseConfigFile should pass 1`] = `
   },
 }
 `;
+
+exports[`write files to temporary dir loadStateFile should pass with existing state file 1`] = `
+"{"format":"base64/gzip","data":"dGVzdA=="}
+"
+`;
diff --git a/airbyte-local-cli-nodejs/test/command.test.ts b/airbyte-local-cli-nodejs/test/command.test.ts
@@ -12,6 +12,7 @@ const defaultConfig = {
   rawMessages: false,
   keepContainers: false,
   logLevel: 'info',
+  debug: false,
 };
 
 afterEach(() => {
@@ -196,6 +197,7 @@ describe('Check other options', () => {
       rawMessages: true,
       keepContainers: true,
       logLevel: 'debug',
+      debug: true,
     });
   });
 
@@ -207,6 +209,7 @@ describe('Check other options', () => {
       src: {image: 'source-image', config: {}},
       dst: {image: 'destination-image', config: {}},
       logLevel: 'debug',
+      debug: true,
     });
   });
 

diff --git a/airbyte-local-cli-nodejs/test/exec/resources/test_config_file.json b/airbyte-local-cli-nodejs/test/exec/resources/test_config_file.json
diff --git a/airbyte-local-cli-nodejs/test/exec/resources/test_config_file_invalid b/airbyte-local-cli-nodejs/test/exec/resources/test_config_file_invalid
diff --git a/airbyte-local-cli-nodejs/test/utils.it.test.ts b/airbyte-local-cli-nodejs/test/utils.it.test.ts
@@ -1,6 +1,16 @@
 import {existsSync, mkdtempSync, readFileSync, rmSync} from 'node:fs';
+import {tmpdir} from 'node:os';
 
-import {checkDockerInstalled, cleanUp, createTmpDir, loadStateFile, parseConfigFile} from '../src/utils';
+import {FarosConfig} from '../src/types';
+import {
+  checkDockerInstalled,
+  cleanUp,
+  createTmpDir,
+  FILENAME_PREFIX,
+  loadStateFile,
+  parseConfigFile,
+  writeConfig,
+} from '../src/utils';
 
 describe('parseConfigFile', () => {
   it('should pass', () => {
@@ -35,31 +45,127 @@ describe('createTmpDir', () => {
   });
 });
 
-describe('loadStateFile', () => {
+describe('write files to temporary dir', () => {
   let tmpDirPath: string;
   beforeAll(() => {
-    tmpDirPath = mkdtempSync('test-temp-dir');
+    tmpDirPath = mkdtempSync(`${tmpdir()}/test-temp-dir`);
   });
   afterAll(() => {
     rmSync(tmpDirPath, {recursive: true, force: true});
   });
 
-  it('should pass without existing state file', () => {
-    expect(() => loadStateFile(tmpDirPath)).not.toThrow();
-    expect(existsSync(`${tmpDirPath}/state.json`)).toBe(true);
-    expect(readFileSync(`${tmpDirPath}/state.json`, 'utf8')).toBe('{}');
-  });
+  describe('loadStateFile', () => {
+    it('should pass without existing state file', () => {
+      expect(() => loadStateFile(tmpDirPath)).not.toThrow();
+      expect(existsSync(`${tmpDirPath}/state.json`)).toBe(true);
+      expect(readFileSync(`${tmpDirPath}/state.json`, 'utf8')).toBe('{}');
+    });
+
+    it('should pass with existing state file', () => {
+      const testStateFile = 'test/resources/test__state.json';
+      expect(() => loadStateFile(tmpDirPath, testStateFile)).not.toThrow();
+      expect(existsSync(`${tmpDirPath}/state.json`)).toBe(true);
+      expect(readFileSync(`${tmpDirPath}/state.json`, 'utf8')).toMatchSnapshot();
+    });
 
-  it('should pass with existing state file', () => {
-    const testStateFile = 'test/resources/test__state.json';
-    expect(() => loadStateFile(tmpDirPath, testStateFile)).not.toThrow();
-    expect(existsSync(`${tmpDirPath}/state.json`)).toBe(true);
-    expect(readFileSync(`${tmpDirPath}/state.json`, 'utf8')).toMatchSnapshot();
+    it('should fail if state file is not loaded', () => {
+      expect(() => loadStateFile(tmpDirPath, 'non-exist-state-file')).toThrow(
+        `State file 'non-exist-state-file' not found. Please make sure the state file exists and have read access.`,
+      );
+    });
   });
 
-  it('should fail if state file is not loaded', () => {
-    expect(() => loadStateFile(tmpDirPath, 'non-exist-state-file')).toThrow(
-      `State file 'non-exist-state-file' not found. Please make sure the state file exists and have read access.`,
-    );
+  describe('writeConfig', () => {
+    const testConfig: FarosConfig = {
+      src: {
+        image: 'farosai/airbyte-test-source',
+        config: {
+          username: 'test',
+          password: 'test',
+          url: 'test',
+        },
+        catalog: {
+          tests: {disabled: true},
+          projects: {disabled: true},
+        },
+      },
+      dst: {
+        image: 'farosai/airbyte-test-destination',
+        config: {
+          edition_config: {
+            graph: 'default',
+            edition: 'cloud',
+            api_url: 'https://test.api.faros.ai',
+          },
+        },
+      },
+
+      // default values
+      srcCheckConnection: false,
+      dstUseHostNetwork: false,
+      srcPull: false,
+      dstPull: false,
+      fullRefresh: false,
+      rawMessages: false,
+      keepContainers: false,
+      logLevel: 'info',
+      debug: false,
+      stateFile: undefined,
+      connectionName: undefined,
+      srcOutputFile: undefined,
+      srcInputFile: undefined,
+    };
+
+    afterEach(() => {
+      rmSync(`${FILENAME_PREFIX}_config.json`, {force: true});
+      rmSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_config.json`, {force: true});
+      rmSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_config.json`, {force: true});
+      rmSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_catalog.json`, {force: true});
+      rmSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_catalog.json`, {force: true});
+    });
+
+    it('should write files', () => {
+      expect(() => writeConfig(tmpDirPath, structuredClone(testConfig))).not.toThrow();
+      expect(existsSync(`${FILENAME_PREFIX}_config.json`)).toBe(true);
+      expect(existsSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_config.json`)).toBe(true);
+      expect(existsSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_config.json`)).toBe(true);
+      expect(existsSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_catalog.json`)).toBe(true);
+      expect(existsSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_catalog.json`)).toBe(true);
+
+      expect(readFileSync(`${FILENAME_PREFIX}_config.json`, 'utf8')).toEqual(
+        JSON.stringify({src: testConfig.src, dst: testConfig.dst}, null, 2),
+      );
+      expect(readFileSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_config.json`, 'utf8')).toEqual(
+        JSON.stringify(testConfig.src?.config, null, 2),
+      );
+      expect(readFileSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_config.json`, 'utf8')).toEqual(
+        JSON.stringify(testConfig.dst?.config, null, 2),
+      );
+      expect(readFileSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_catalog.json`, 'utf8')).toEqual(
+        JSON.stringify(testConfig.src?.catalog, null, 2),
+      );
+      expect(readFileSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_catalog.json`, 'utf8')).toEqual(
+        JSON.stringify(testConfig.src?.catalog, null, 2),
+      );
+    });
+
+    it('should alter config if debug is enabled', () => {
+      const testConfigDebug = {...structuredClone(testConfig), debug: true};
+      (testConfigDebug.src as any).image = 'farosai/airbyte-faros-feeds-source:v1';
+      expect(() => writeConfig(tmpDirPath, structuredClone(testConfigDebug))).not.toThrow();
+      expect(existsSync(`${FILENAME_PREFIX}_config.json`)).toBe(true);
+      expect(existsSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_config.json`)).toBe(true);
+      expect(existsSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_config.json`)).toBe(true);
+
+      expect(readFileSync(`${FILENAME_PREFIX}_config.json`, 'utf8')).toEqual(
+        JSON.stringify({src: testConfigDebug.src, dst: testConfigDebug.dst}, null, 2),
+      );
+      expect(readFileSync(`${tmpDirPath}/${FILENAME_PREFIX}_src_config.json`, 'utf8')).toEqual(
+        JSON.stringify({...testConfigDebug.src?.config, feed_cfg: {debug: true}}, null, 2),
+      );
+      expect(readFileSync(`${tmpDirPath}/${FILENAME_PREFIX}_dst_config.json`, 'utf8')).toEqual(
+        JSON.stringify(testConfigDebug.dst?.config, null, 2),
+      );
+    });
   });
 });