From 86c8200ca1b3b4ff1f850ce2982151c65843ecc7 Mon Sep 17 00:00:00 2001 From: Eunjae Lee Date: Tue, 29 Mar 2022 16:47:56 +0200 Subject: [PATCH 1/3] chore(ci): fix regression regarding emptyDir (#306) * chore(ci): fix regression regarding emptyDir * chore: fix import order problem * Update scripts/release/process-release.ts Co-authored-by: Pierre Millot Co-authored-by: Pierre Millot --- scripts/release/process-release.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/release/process-release.ts b/scripts/release/process-release.ts index 5c4b09d44b..0bdda86922 100755 --- a/scripts/release/process-release.ts +++ b/scripts/release/process-release.ts @@ -1,9 +1,10 @@ /* eslint-disable no-console */ import fsp from 'fs/promises'; +import path from 'path'; import dotenv from 'dotenv'; import execa from 'execa'; -import { emptyDir, copy } from 'fs-extra'; +import { copy, remove } from 'fs-extra'; import semver from 'semver'; import type { ReleaseType } from 'semver'; @@ -110,6 +111,14 @@ async function updateOpenApiTools( ); } +async function emptyDirExceptForDotGit(dir: string): Promise { + for (const file of await fsp.readdir(dir)) { + if (file !== '.git') { + await remove(path.resolve(dir, file)); + } + } +} + async function updateChangelog({ lang, issueBody, @@ -206,7 +215,7 @@ async function processRelease(): Promise { }); const clientPath = toAbsolutePath(getLanguageFolder(lang)); - await emptyDir(tempGitDir); + await emptyDirExceptForDotGit(tempGitDir); await copy(clientPath, tempGitDir, { preserveTimestamps: true }); await configureGitHubAuthor(tempGitDir); From 7a7164acd853797c262affd94d936c466bb1f607 Mon Sep 17 00:00:00 2001 From: Eunjae Lee Date: Wed, 30 Mar 2022 11:44:33 +0200 Subject: [PATCH 2/3] chore(ci): replace github token for processing release (#310) --- .github/workflows/process-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/process-release.yml b/.github/workflows/process-release.yml index 0e2efd57d0..6614e0cb00 100644 --- a/.github/workflows/process-release.yml +++ b/.github/workflows/process-release.yml @@ -14,6 +14,7 @@ jobs: - uses: actions/checkout@v2 with: fetch-depth: 0 + token: ${{ secrets.TOKEN_RELEASE_BOT }} - name: Setup id: setup From 7a65dace61dfe6a8aedb105cc23c5b16a7911a74 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Cl=C3=A9ment=20Vannicatte?= <20689156+shortcuts@users.noreply.github.com> Date: Wed, 30 Mar 2022 15:48:00 +0200 Subject: [PATCH 3/3] chore: cache custom generators (#311) --- .gitignore | 1 + scripts/buildSpecs.ts | 37 ++++++++------------- scripts/common.ts | 77 ++++++++++++++++++++++++++++++++++++++++++- scripts/types.ts | 13 ++++++++ 4 files changed, 104 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index a41aa410dd..662cf6f91d 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ build pom.xml dist +.cache .openapi-generator diff --git a/scripts/buildSpecs.ts b/scripts/buildSpecs.ts index 022449cd74..1f211c9f46 100644 --- a/scripts/buildSpecs.ts +++ b/scripts/buildSpecs.ts @@ -1,9 +1,8 @@ import fsp from 'fs/promises'; -import { hashElement } from 'folder-hash'; import yaml from 'js-yaml'; -import { exists, run, toAbsolutePath } from './common'; +import { checkForCache, exists, run, toAbsolutePath } from './common'; import { createSpinner } from './oraLog'; import type { Spec } from './pre-gen/setHostsOptions'; @@ -53,30 +52,22 @@ async function buildSpec( let hash = ''; if (useCache) { - const spinner = createSpinner( - `checking cache for '${client}'`, + const { cacheExists, hash: newCache } = await checkForCache( + { + job: `'${client}' specs`, + folder: toAbsolutePath('specs/'), + generatedFiles: [`bundled/${client}.yml`], + filesToCache: [client, 'common'], + cacheFile, + }, verbose - ).start(); - // check if file and cache exist - if (await exists(toAbsolutePath(`specs/bundled/${client}.yml`))) { - // compare with stored cache - const specHash = (await hashElement(toAbsolutePath(`specs/${client}`))) - .hash; - const commonHash = (await hashElement(toAbsolutePath(`specs/common`))) - .hash; - hash = `${specHash}-${commonHash}`; - if (await exists(cacheFile)) { - const storedHash = (await fsp.readFile(cacheFile)).toString(); - if (storedHash === hash) { - spinner.succeed( - `skipped building ${client} spec because the files did not change` - ); - return; - } - } + ); + + if (cacheExists) { + return; } - spinner.info(`cache not found for ${client}' spec`); + hash = newCache; } const spinner = createSpinner(`building ${client} spec`, verbose).start(); diff --git a/scripts/common.ts b/scripts/common.ts index ba8ec31f33..a01c6165a0 100644 --- a/scripts/common.ts +++ b/scripts/common.ts @@ -2,11 +2,17 @@ import fsp from 'fs/promises'; import path from 'path'; import execa from 'execa'; // https://github.com/sindresorhus/execa/tree/v5.1.1 +import { hashElement } from 'folder-hash'; import openapitools from '../openapitools.json'; import { createSpinner } from './oraLog'; -import type { Generator, RunOptions } from './types'; +import type { + CheckForCache, + CheckForCacheOptions, + Generator, + RunOptions, +} from './types'; export const CI = Boolean(process.env.CI); export const DOCKER = Boolean(process.env.DOCKER); @@ -194,11 +200,80 @@ export async function gitCommit({ ); } +export async function checkForCache( + { + job, + folder, + generatedFiles, + filesToCache, + cacheFile, + }: CheckForCacheOptions, + verbose: boolean +): Promise { + const spinner = createSpinner(`checking cache for ${job}`, verbose).start(); + const cache: CheckForCache = { + cacheExists: false, + hash: '', + }; + const generatedFilesExists = ( + await Promise.all( + generatedFiles.map((generatedFile) => + exists(`${folder}/${generatedFile}`) + ) + ) + ).every((exist) => exist); + + for (const fileToCache of filesToCache) { + const fileHash = (await hashElement(`${folder}/${fileToCache}`)).hash; + + cache.hash = `${cache.hash}-${fileHash}`; + } + + // We only skip if both the cache and the generated file exists + if (generatedFilesExists && (await exists(cacheFile))) { + const storedHash = (await fsp.readFile(cacheFile)).toString(); + if (storedHash === cache.hash) { + spinner.succeed(`job skipped, cache found for ${job}`); + return { + cacheExists: true, + hash: cache.hash, + }; + } + } + + spinner.info(`cache not found for ${job}`); + + return cache; +} + export async function buildCustomGenerators(verbose: boolean): Promise { + const cacheFile = toAbsolutePath('generators/.cache'); + const { cacheExists, hash } = await checkForCache( + { + job: 'custom generators', + folder: toAbsolutePath('generators/'), + generatedFiles: ['build'], + filesToCache: ['src', 'build.gradle', 'settings.gradle'], + cacheFile, + }, + verbose + ); + + if (cacheExists) { + return; + } + const spinner = createSpinner('building custom generators', verbose).start(); + await run('./gradle/gradlew --no-daemon -p generators assemble', { verbose, }); + + if (hash) { + spinner.text = 'storing custom generators cache'; + await fsp.writeFile(cacheFile, hash); + } + spinner.succeed(); } diff --git a/scripts/types.ts b/scripts/types.ts index 3b2382a58e..31aff2867a 100644 --- a/scripts/types.ts +++ b/scripts/types.ts @@ -1,5 +1,18 @@ import type config from '../config/clients.config.json'; +export type CheckForCacheOptions = { + job: string; + folder: string; + generatedFiles: string[]; + filesToCache: string[]; + cacheFile: string; +}; + +export type CheckForCache = { + cacheExists: boolean; + hash: string; +}; + export type Generator = Record & { language: string; client: string;