diff --git a/src/commands/hld/reconcile.test.ts b/src/commands/hld/reconcile.test.ts index 4d50ca2cd..ea91dfd05 100644 --- a/src/commands/hld/reconcile.test.ts +++ b/src/commands/hld/reconcile.test.ts @@ -16,11 +16,14 @@ import { execute, getFullPathPrefix, normalizedName, + purgeRepositoryComponents, ReconcileDependencies, reconcileHld, testAndGetAbsPath, validateInputs, } from "./reconcile"; +import mockFs from "mock-fs"; +import fs from "fs"; beforeAll(() => { enableVerboseLogging(); @@ -169,6 +172,109 @@ describe("createAccessYaml", () => { }); }); +describe("purgeRepositoryComponents", () => { + const fsSpy = jest.spyOn(fs, "unlink"); + + beforeEach(() => { + mockFs({ + "hld-repo": { + config: { + "common.yaml": "someconfigfile", + }, + "bedrock-project-repo": { + "access.yaml": "someaccessfile", + config: { + "common.yaml": "someconfigfile", + }, + serviceA: { + config: { + "common.yaml": "someconfigfile", + }, + master: { + config: { + "common.yaml": "someconfigfile", + "prod.yaml": "someconfigfile", + "stage.yaml": "someconfigfile", + }, + static: { + "ingressroute.yaml": "ingressroutefile", + "middlewares.yaml": "middlewaresfile", + }, + "component.yaml": "somecomponentfile", + }, + "component.yaml": "somecomponentfile", + }, + "component.yaml": "somecomponentfile", + }, + "component.yaml": "somecomponentfile", + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + jest.clearAllMocks(); + fsSpy.mockClear(); + }); + + const hldPath = "hld-repo"; + const repositoryName = "bedrock-project-repo"; + + it("should invoke fs.unlink for each file in project repository except config files and access.yaml", async () => { + purgeRepositoryComponents(hldPath, repositoryName); + + expect(fs.unlink).toHaveBeenCalledTimes(5); + expect(fs.unlink).toHaveBeenCalledWith( + path.join(hldPath, repositoryName, "component.yaml"), + expect.any(Function) + ); + expect(fs.unlink).toHaveBeenCalledWith( + path.join(hldPath, repositoryName, "serviceA", "component.yaml"), + expect.any(Function) + ); + expect(fs.unlink).toHaveBeenCalledWith( + path.join( + hldPath, + repositoryName, + "serviceA", + "master", + "component.yaml" + ), + expect.any(Function) + ); + expect(fs.unlink).toHaveBeenCalledWith( + path.join( + hldPath, + repositoryName, + "serviceA", + "master", + "static", + "middlewares.yaml" + ), + expect.any(Function) + ); + expect(fs.unlink).toHaveBeenCalledWith( + path.join( + hldPath, + repositoryName, + "serviceA", + "master", + "static", + "ingressroute.yaml" + ), + expect.any(Function) + ); + }); + + it("should throw an error if fs fails", async () => { + fsSpy.mockImplementationOnce(() => { + throw Error("some error"); + }); + + expect(() => purgeRepositoryComponents(hldPath, repositoryName)).toThrow(); + }); +}); + describe("createRepositoryComponent", () => { let exec = jest.fn().mockReturnValue(Promise.resolve({})); const hldPath = `myMonoRepo`; @@ -446,6 +552,7 @@ describe("reconcile tests", () => { exec: jest.fn().mockReturnValue(Promise.resolve({})), generateAccessYaml: jest.fn(), getGitOrigin: jest.fn(), + purgeRepositoryComponents: jest.fn(), writeFile: jest.fn(), }; @@ -494,6 +601,7 @@ describe("reconcile tests", () => { expect(dependencies.createMiddlewareForRing).toHaveBeenCalledTimes(2); expect(dependencies.createIngressRouteForRing).toHaveBeenCalledTimes(2); expect(dependencies.generateAccessYaml).toHaveBeenCalledTimes(1); + expect(dependencies.purgeRepositoryComponents).toHaveBeenCalledTimes(1); expect(dependencies.generateAccessYaml).toBeCalledWith( "path/to/hld/service", git, diff --git a/src/commands/hld/reconcile.ts b/src/commands/hld/reconcile.ts index e715f97ca..5bdc5b0e1 100644 --- a/src/commands/hld/reconcile.ts +++ b/src/commands/hld/reconcile.ts @@ -19,6 +19,7 @@ import { BedrockFile, BedrockServiceConfig } from "../../types"; import decorator from "./reconcile.decorator.json"; import { build as buildError, log as logError } from "../../lib/errorBuilder"; import { errorStatusCode } from "../../lib/errorStatusCode"; +import { getAllFilesInDirectory } from "../../lib/ioUtil"; /** * IExecResult represents the possible return value of a Promise based wrapper @@ -101,6 +102,46 @@ type MiddlewareMap>> = { default?: T; }; +/** + * In spk hld reconcile, the results should always result in the same artifacts being created based on the state of bedrock.yaml. + * The only exception is for files under the /config directories and any access.yaml files. + * @param absHldPath Absolute path to the local HLD repository directory + * @param repositoryName Name of the bedrock project repository/directory inside of the HLD repository + */ +export const purgeRepositoryComponents = ( + absHldPath: string, + repositoryName: string +): void => { + assertIsStringWithContent(absHldPath, "hld-path"); + assertIsStringWithContent(repositoryName, "repository-name"); + + const filesToDelete = getAllFilesInDirectory( + path.join(absHldPath, repositoryName) + ).filter( + (filePath) => + !filePath.endsWith("access.yaml") && !filePath.match(/config\/.*\.yaml$/) + ); + + try { + filesToDelete.forEach((file) => { + fs.unlink(file, function (err) { + if (err) throw err; + console.log(`${file} deleted!`); + }); + }); + } catch (err) { + throw buildError( + errorStatusCode.FILE_IO_ERR, + { + errorKey: "hld-reconcile-err-purge-repo-comps", + values: [repositoryName, absHldPath], + }, + err + ); + } + return; +}; + export const createRepositoryComponent = async ( execCmd: typeof execAndLog, absHldPath: string, @@ -381,6 +422,7 @@ export interface ReconcileDependencies { getGitOrigin: typeof tryGetGitOrigin; generateAccessYaml: typeof generateAccessYaml; createAccessYaml: typeof createAccessYaml; + purgeRepositoryComponents: typeof purgeRepositoryComponents; createRepositoryComponent: typeof createRepositoryComponent; configureChartForRing: typeof configureChartForRing; createServiceComponent: typeof createServiceComponent; @@ -454,6 +496,12 @@ export const reconcileHld = async ( ): Promise => { const { services: managedServices, rings: managedRings } = bedrockYaml; + // To support removing services and rings, first remove all files under an application repository directory except anything in a /config directory and any access.yaml files, then we generate all values again. + dependencies.purgeRepositoryComponents( + absHldPath, + normalizedName(repositoryName) + ); + // Create Repository Component if it doesn't exist. // In a pipeline, the repository component is the name of the application repository. await dependencies.createRepositoryComponent( @@ -649,6 +697,7 @@ export const execute = async ( exec: execAndLog, generateAccessYaml, getGitOrigin: tryGetGitOrigin, + purgeRepositoryComponents, writeFile: fs.writeFileSync, }; diff --git a/src/lib/i18n.json b/src/lib/i18n.json index bbb0948de..413259293 100644 --- a/src/lib/i18n.json +++ b/src/lib/i18n.json @@ -31,6 +31,7 @@ "hld-reconcile-err-path": "Could not validate {0} path.", "hld-reconcile-err-cmd-failed": "An error occurred executing command: {0}", "hld-reconcile-err-cmd-exe": "An error occurred while reconciling HLD.", + "hld-reconcile-err-purge-repo-comps": "Could not purge hld repository component {0} in path {1}.", "infra-scaffold-cmd-failed": "Infra scaffold command was not successfully executed.", "infra-scaffold-cmd-src-missing": "Value for source is required because it cannot be constructed with properties in spk-config.yaml. Provide value for source.", diff --git a/src/lib/ioUtil.test.ts b/src/lib/ioUtil.test.ts index cef6c7fc3..ae7c24d6b 100644 --- a/src/lib/ioUtil.test.ts +++ b/src/lib/ioUtil.test.ts @@ -3,10 +3,25 @@ import path from "path"; import uuid from "uuid/v4"; import { createTempDir, + getAllFilesInDirectory, getMissingFilenames, isDirEmpty, removeDir, } from "./ioUtil"; +import mockFs from "mock-fs"; +import { disableVerboseLogging, enableVerboseLogging, logger } from "../logger"; + +beforeAll(() => { + enableVerboseLogging(); +}); + +afterAll(() => { + disableVerboseLogging(); +}); + +beforeEach(() => { + jest.clearAllMocks(); +}); describe("test createTempDir function", () => { it("create and existence check", () => { @@ -110,3 +125,69 @@ describe("test doFilesExist function", () => { expect(missing.length).toBe(1); }); }); + +describe("test getAllFilesInDirectory function", () => { + beforeEach(() => { + mockFs({ + "hld-repo": { + config: { + "common.yaml": "someconfigfile", + }, + "bedrock-project-repo": { + "access.yaml": "someaccessfile", + config: { + "common.yaml": "someconfigfile", + }, + serviceA: { + config: { + "common.yaml": "someconfigfile", + }, + master: { + config: { + "common.yaml": "someconfigfile", + "prod.yaml": "someconfigfile", + "stage.yaml": "someconfigfile", + }, + static: { + "ingressroute.yaml": "ingressroutefile", + "middlewares.yaml": "middlewaresfile", + }, + "component.yaml": "somecomponentfile", + }, + "component.yaml": "somecomponentfile", + }, + "component.yaml": "somecomponentfile", + }, + "component.yaml": "somecomponentfile", + }, + }); + }); + + afterEach(() => { + mockFs.restore(); + }); + + it("gets all files in a populated directory", () => { + const fileList = getAllFilesInDirectory("hld-repo"); + expect(fileList).toHaveLength(13); + expect(fileList).toContain( + "hld-repo/bedrock-project-repo/serviceA/master/static/middlewares.yaml" + ); + + const filesToDelete = fileList.filter( + (filePath) => + !filePath.match(/access\.yaml$/) && !filePath.match(/config\/.*\.yaml$/) + ); + logger.info("filestoDelete.length: " + filesToDelete.length); + logger.info(filesToDelete); + }); + + it("returns an empty list when there's no files directory", () => { + mockFs({ + emptyDirectory: {}, + }); + + const fileList = getAllFilesInDirectory("emptyDirectory"); + expect(fileList).toHaveLength(0); + }); +}); diff --git a/src/lib/ioUtil.ts b/src/lib/ioUtil.ts index fc8dd4168..606a180f9 100644 --- a/src/lib/ioUtil.ts +++ b/src/lib/ioUtil.ts @@ -2,6 +2,7 @@ import fs from "fs"; import os from "os"; import path from "path"; import uuid from "uuid/v4"; +import { logger } from "../logger"; /** * Creates a random directory in tmp directory. @@ -68,3 +69,27 @@ export const getMissingFilenames = ( ) .map((f) => path.basename(f)); }; + +/** + * Returns all files/filepaths in this directory and subdirectories. + * + * @param dir directory to search + * @param files list of files + * @returns list of files in given directory and subdirectories. + */ +export const getAllFilesInDirectory = ( + dir: string, + files: string[] = [] +): string[] => { + const filesInDir = fs.readdirSync(dir); + + filesInDir.forEach(function (file) { + if (fs.statSync(path.join(dir, file)).isDirectory()) { + files = getAllFilesInDirectory(path.join(dir, file), files); + } else { + files.push(path.join(dir, file)); + } + }); + + return files; +};