improve sass building workflow and optimize bot script (finos#665)

* update PR template

* revamp sass dev and build workflow

* move github bot/automation to its own package

* bump dependencies

* add changeset

* exclude bot package from checks

.changeset/chilly-olives-drum.md

@@ -0,0 +1,23 @@
+---
+'@finos/legend-application': patch
+'@finos/legend-art': patch
+'@finos/legend-dev-utils': patch
+'@finos/legend-extension-dsl-data-space': patch
+'@finos/legend-extension-dsl-diagram': patch
+'@finos/legend-extension-dsl-serializer': patch
+'@finos/legend-extension-dsl-text': patch
+'@finos/legend-extension-external-format-json-schema': patch
+'@finos/legend-extension-external-store-service': patch
+'@finos/legend-graph': patch
+'@finos/legend-query': patch
+'@finos/legend-query-app': patch
+'@finos/legend-query-deployment': patch
+'@finos/legend-server-depot': patch
+'@finos/legend-server-sdlc': patch
+'@finos/legend-studio': patch
+'@finos/legend-studio-app': patch
+'@finos/legend-studio-deployment': patch
+'@finos/legend-studio-extension-management-toolkit': patch
+'@finos/legend-studio-extension-query-builder': patch
+'@finos/legend-github-bot': patch
+---

.changeset/good-snails-collect.md

@@ -0,0 +1,20 @@
+---
+'@finos/legend-application': patch
+'@finos/legend-art': patch
+'@finos/legend-dev-utils': patch
+'@finos/legend-extension-dsl-data-space': patch
+'@finos/legend-extension-dsl-diagram': patch
+'@finos/legend-extension-dsl-serializer': patch
+'@finos/legend-extension-dsl-text': patch
+'@finos/legend-extension-external-format-json-schema': patch
+'@finos/legend-extension-external-store-service': patch
+'@finos/legend-graph': patch
+'@finos/legend-query': patch
+'@finos/legend-query-app': patch
+'@finos/legend-server-depot': patch
+'@finos/legend-server-sdlc': patch
+'@finos/legend-studio': patch
+'@finos/legend-studio-app': patch
+'@finos/legend-studio-extension-management-toolkit': patch
+'@finos/legend-studio-extension-query-builder': patch
+---

.github/PULL_REQUEST_TEMPLATE.md

@@ -23,6 +23,8 @@
 ## How did you test this change?
 
 - [ ] Test(s) added
+- [ ] Manual testing (please provide screenshots/recordings)
+- [ ] No testing (please provide an explanation)
 
 <!--
   Demonstrate the code is solid. Screenshots/videos if the pull request changes the user interface.

.github/workflows/bot-new-issue-label.yml

@@ -26,12 +26,16 @@ jobs:
         with:
           node-version: 14.x
       - name: Install dependencies
-        run: yarn
+        working-directory: ./scripts/bot
+        # Avoid Yarn failing installation due to modified lock file
+        # See https://github.com/finos/legend-studio/tree/master/scripts/bot
+        run: yarn --immutable=false
       - name: Label new issue
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           ORGANIZATION_READ_TOKEN: ${{ secrets.READ_ORG_TOKEN }}
           ORGANIZATION_NAME: finos
           TEAM_NAME: legend-contributors
           TEAM_LABEL: 'Studio Core Team'
-        run: node ./scripts/workflow/labelNewIssue.js
+        working-directory: ./scripts/bot
+        run: node ./labelNewIssue.js

CONTRIBUTING.md

@@ -124,37 +124,20 @@ Now, you're good to start. After the setup step, you can start the application y
 yarn dev # alias: `yarn start` - this by default will start Studio
 ```
 
-Each workspace in the monorepo should have a `dev` script. Run these (in separate terminal tabs) when you are making changes in these workspaces to rebuild on change. Otherwise, after making change, you have to manually rebuild the workspace using the `build` script. Following are some useful scripts for development.
+Each workspace in the monorepo should have a `dev` script. Run these (in separate terminal tabs) when you are making changes in these workspaces to rebuild on change. However, this requires building packages following the dependency order. As such, we recommend the following workflow using multiple terminal tabs:
 
 ```sh
-# Clean and build all workspaces.
-#
-# NOTE: To save time, keep the build scope as close to your changes as possible.
-#
-# In general, you can call a script from a particular workspace using
-# the following command construct
-#   yarn workspace <workspace-name> <workspace-script>
-# e.g. To call `build` script of workspace `lib1`:
-#   yarn workspace lib1 build
-#
-# There are also other build scripts within each workspace which targets a smaller build scopes.
-yarn build
-
-# Using project reference, build and develop Typescript files in all workspaces.
-# This is usually sufficient if you make code change not related to styling.
-yarn dev:tsc
-```
-
-As such, the recommended workflow is to have multiple terminal tabs opened
-
-```sh
-# one for the application served with Webpack
+# keep one terminal tab for auto-building and serving the web application using Webpack
 yarn dev
 
-# another tab is for Typescript compiler
-# NOTE: this process could take some time in certain cases
-# so if you need to fine tune, you can use `yarn workspace ... dev` instead
+# have other terminal tabs for auto-building code and style
 yarn dev:tsc
+yarn dev:sass
+
+# NOTE: Sometimes, it might be better to call `dev` for each workspace, either `cd`
+# to the workspace and run the script, or using `yarn workspace <workspace-name> <workspace-script>`
+# syntax to call the script from anywhere in the project.
+# e.g. To call `build` script of workspace `lib1`, use the command `yarn workspace lib1 build`
 ```
 
 #### :construction: Testing your code

package.json

@@ -3,7 +3,8 @@
   "private": true,
   "type": "module",
   "workspaces": [
-    "packages/*"
+    "packages/*",
+    "scripts/github-bot"
   ],
   "scripts": {
     "build": "yarn clean && yarn workspaces foreach --topological-dev --verbose run build",
@@ -23,12 +24,13 @@
     "clean": "rimraf \"build\" && yarn workspaces foreach --all --topological-dev --parallel --verbose run clean",
     "dev": "yarn dev:studio",
     "dev:query": "yarn workspace @finos/legend-query-deployment dev",
+    "dev:sass": "node ./scripts/workflow/devSassAll.js",
     "dev:studio": "yarn workspace @finos/legend-studio-deployment dev",
     "dev:tsc": "tsc --build --watch",
     "fix": "yarn fix:copyright && yarn fix:style && yarn fix:js && yarn fix:format",
     "fix:copyright": "node ./scripts/copyright/checkCopyrightHeaders.js --update",
     "fix:format": "prettier --write --loglevel=warn \"(scripts|docs)/**/*.{md,json,mjs,cjs,js,ts,tsx,html,scss,css}\" \"packages/*/**/*.{md,json,mjs,cjs,js,ts,tsx,html,scss,css}\" && yarn sort-package-json \"package.json\" \"packages/*/package.json\"",
-    "fix:js": "cross-env NODE_ENV=production FORCE_COLOR=1 eslint --cache --cache-location ./build/.eslintcache \"scripts/**/*.{mjs,cjs,js,ts,tsx}\" --report-unused-disable-directives --fix && cross-env FORCE_COLOR=1 yarn workspaces foreach --all --topological-dev --parallel --exclude legend-studio --verbose run lint:js --fix",
+    "fix:js": "cross-env NODE_ENV=production FORCE_COLOR=1 eslint --cache --cache-location ./build/.eslintcache \"scripts/**/*.{mjs,cjs,js,ts,tsx}\" --report-unused-disable-directives --fix && cross-env FORCE_COLOR=1 yarn workspaces foreach --all --topological-dev --parallel --exclude legend-studio --exclude \"@finos/legend-internal-*\" --verbose run lint:js --fix",
     "fix:style": "yarn lint:style --fix",
     "git:install-hooks": "husky install",
     "git:pre-commit": "lint-staged",
@@ -37,7 +39,7 @@
     "postinstall": "husky install",
     "lint": "yarn lint:js && yarn lint:style",
     "lint:ci": "yarn lint:js --max-warnings=0 && yarn lint:style --max-warnings=0",
-    "lint:js": "cross-env NODE_ENV=production FORCE_COLOR=1 eslint --cache --cache-location ./build/.eslintcache \"scripts/**/*.{mjs,cjs,js,ts,tsx}\" --report-unused-disable-directives && cross-env FORCE_COLOR=1 yarn workspaces foreach --all --topological-dev --parallel --verbose run lint:js",
+    "lint:js": "cross-env NODE_ENV=production FORCE_COLOR=1 eslint --cache --cache-location ./build/.eslintcache \"scripts/**/*.{mjs,cjs,js,ts,tsx}\" --report-unused-disable-directives && cross-env FORCE_COLOR=1 yarn workspaces foreach --all --topological-dev --parallel --exclude legend-studio --exclude \"@finos/legend-internal-*\" --verbose run lint:js",
     "lint:style": "stylelint --cache --cache-location ./build/.stylelintcache \"packages/*/style/**/*.{scss,css}\"",
     "publish:prepare": "cross-env FORCE_COLOR=1 yarn build && cross-env FORCE_COLOR=1 yarn workspaces foreach --all --no-private --topological-dev --parallel --verbose run publish:prepare",
     "publish:snapshot": "yarn publish:prepare && cross-env FORCE_COLOR=1 yarn workspaces foreach --all --no-private --topological-dev --parallel --verbose run publish:snapshot",
@@ -92,6 +94,7 @@
     "lint-staged": "11.2.6",
     "prettier": "2.4.1",
     "rimraf": "3.0.2",
+    "sass": "1.43.4",
     "semver": "7.3.5",
     "sort-package-json": "1.53.1",
     "stylelint": "14.0.1",

packages/legend-application/_package.config.js

@@ -14,20 +14,11 @@
  * limitations under the License.
  */
 
-import { generateBundleCopyrightText } from '../../scripts/copyright/PackageCopyrightHelper.js';
-import { dirname } from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
 export default {
   publish: {
     typescript: {
       main: './tsconfig.build.json',
       others: ['./tsconfig.package.json'],
     },
   },
-  build: {
-    copyrightText: generateBundleCopyrightText(__dirname),
-  },
 };

packages/legend-application/package.json

@@ -25,11 +25,11 @@
   "types": "lib/index.d.ts",
   "scripts": {
     "build": "yarn clean && yarn build:sass && yarn build:tsc",
-    "build:sass": "cross-env INIT_CWD=$INIT_CWD sass ./style/index.scss ./lib/index.css --style=compressed && node ../../scripts/copyright/addBundledCodeCopyrightHeader.js ./lib/index.css",
+    "build:sass": "cross-env INIT_CWD=$INIT_CWD node ../../scripts/workflow/buildSass.js",
     "build:tsc": "tsc --project ./tsconfig.build.json",
     "clean": "rimraf \"lib\" \"build\"",
     "dev": "npm-run-all --parallel dev:sass dev:tsc",
-    "dev:sass": "sass ./style/index.scss ./lib/index.css --watch",
+    "dev:sass": "sass style lib --watch",
     "dev:tsc": "tsc --watch --preserveWatchOutput",
     "lint:js": "cross-env NODE_ENV=production eslint --cache --cache-location ./build/.eslintcache --report-unused-disable-directives --parser-options=project:\"./tsconfig.json\" \"./src/**/*.{js,ts,tsx}\"",
     "publish:prepare": "node ../../scripts/release/preparePublishContent.js",

packages/legend-art/_package.config.js

@@ -14,19 +14,10 @@
  * limitations under the License.
  */
 
-import { generateBundleCopyrightText } from '../../scripts/copyright/PackageCopyrightHelper.js';
-import { dirname } from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
 export default {
   publish: {
     typescript: {
       main: './tsconfig.build.json',
     },
   },
-  build: {
-    copyrightText: generateBundleCopyrightText(__dirname),
-  },
 };

packages/legend-art/package.json

@@ -27,11 +27,11 @@
   "types": "lib/index.d.ts",
   "scripts": {
     "build": "yarn clean && yarn build:sass && yarn build:tsc",
-    "build:sass": "cross-env INIT_CWD=$INIT_CWD sass ./style/index.scss ./lib/index.css --style=compressed && node ../../scripts/copyright/addBundledCodeCopyrightHeader.js ./lib/index.css",
+    "build:sass": "cross-env INIT_CWD=$INIT_CWD node ../../scripts/workflow/buildSass.js",
     "build:tsc": "tsc --project ./tsconfig.build.json",
     "clean": "rimraf \"lib\" \"build\"",
     "dev": "npm-run-all --parallel dev:sass dev:tsc",
-    "dev:sass": "sass ./style/index.scss ./lib/index.css --watch",
+    "dev:sass": "sass style lib --watch",
     "dev:tsc": "tsc --watch --preserveWatchOutput",
     "lint:js": "cross-env NODE_ENV=production eslint --cache --cache-location ./build/.eslintcache --report-unused-disable-directives --parser-options=project:\"./tsconfig.json\" \"./src/**/*.{js,ts,tsx}\"",
     "publish:prepare": "node ../../scripts/release/preparePublishContent.js",

packages/legend-dev-utils/CopyrightUtils.js

@@ -14,19 +14,13 @@
  * limitations under the License.
  */
 
-import { resolve } from 'path';
-import { existsSync, lstatSync, writeFile, writeFileSync } from 'fs';
-import { EOL, platform } from 'os';
+import { existsSync, lstatSync, writeFile } from 'fs';
+import { EOL } from 'os';
 import micromatch from 'micromatch';
 import { execSync } from 'child_process';
 import { isBinaryFileSync } from 'isbinaryfile';
 import chalk from 'chalk';
-import {
-  getFileContent,
-  createRegExp,
-  exitWithError,
-  exitWithSuccess,
-} from './DevUtils.js';
+import { getFileContent, createRegExp, exitWithError } from './DevUtils.js';
 
 const GENERIC_INCLUDE_PATTERNS = [
   /\.[^/]+$/, // files with extension
@@ -228,42 +222,3 @@ export const updateCopyrightHeaders = async ({
     console.log('All files look good!');
   }
 };
-
-export const addCopyrightHeaderToBundledOutput = async ({
-  basePath,
-  configPath,
-  file,
-}) => {
-  // NOTE: Windows requires prefix `file://` for absolute path
-  const config = (
-    await import(`${platform() === 'win32' ? 'file://' : ''}${configPath}`)
-  ).default;
-  const copyrightText = config?.build?.copyrightText;
-  if (!copyrightText) {
-    exitWithError(
-      `'build.copyrightText' is not specified in config file: ${configPath}`,
-    );
-  }
-  const bundledOutputFile = resolve(basePath, file);
-  if (!existsSync(bundledOutputFile)) {
-    exitWithError(
-      `Can't find bundled output file '${bundledOutputFile}'. Make sure to build before running this script`,
-    );
-  }
-
-  writeFileSync(
-    bundledOutputFile,
-    `${copyrightText}\n\n${getFileContent(bundledOutputFile)}`,
-    (err) => {
-      exitWithError(
-        `Failed to add copyright header to bundled output file: ${bundledOutputFile}. Error:\n${
-          err.message || err
-        }`,
-      );
-    },
-  );
-
-  exitWithSuccess(
-    `Added copyright header to bundled output file: ${bundledOutputFile}`,
-  );
-};

packages/legend-dev-utils/DevUtils.js

@@ -14,6 +14,7 @@
  * limitations under the License.
  */
 
+import { platform } from 'os';
 import { readFileSync } from 'fs';
 
 export const getFileContent = (file) =>
@@ -37,3 +38,7 @@ export const exitOrThrowError = (msg, throwError = true) => {
 // NOTE: unlike `require`, ESM `import` does not support JSON files without the flag --experimental-json-modules
 // being specified, which is not convenient at all in our setup. So we will use the following approach to load them
 export const loadJSON = (path) => JSON.parse(getFileContent(path));
+
+export const loadJSModule = (path) =>
+  // NOTE: Windows requires prefix `file://` for absolute path
+  import(`${platform() === 'win32' ? 'file://' : ''}${path}`);

packages/legend-extension-dsl-data-space/_package.config.js

@@ -14,20 +14,11 @@
  * limitations under the License.
  */
 
-import { generateBundleCopyrightText } from '../../scripts/copyright/PackageCopyrightHelper.js';
-import { dirname } from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
 export default {
   publish: {
     typescript: {
       main: './tsconfig.build.json',
       others: ['./tsconfig.package.json'],
     },
   },
-  build: {
-    copyrightText: generateBundleCopyrightText(__dirname),
-  },
 };

packages/legend-extension-dsl-data-space/package.json

@@ -25,11 +25,11 @@
   "types": "lib/index.d.ts",
   "scripts": {
     "build": "yarn clean && yarn build:sass && yarn build:tsc",
-    "build:sass": "cross-env INIT_CWD=$INIT_CWD sass ./style/index.scss ./lib/index.css --style=compressed && node ../../scripts/copyright/addBundledCodeCopyrightHeader.js ./lib/index.css",
+    "build:sass": "cross-env INIT_CWD=$INIT_CWD node ../../scripts/workflow/buildSass.js",
     "build:tsc": "tsc --project ./tsconfig.build.json",
     "clean": "rimraf \"lib\" \"build\"",
     "dev": "npm-run-all --parallel dev:sass dev:tsc",
-    "dev:sass": "sass ./style/index.scss ./lib/index.css --watch",
+    "dev:sass": "sass style lib --watch",
     "dev:tsc": "tsc --watch --preserveWatchOutput",
     "lint:js": "cross-env NODE_ENV=production eslint --cache --cache-location ./build/.eslintcache --report-unused-disable-directives --parser-options=project:\"./tsconfig.json\" \"./src/**/*.{js,ts,tsx}\"",
     "publish:prepare": "node ../../scripts/release/preparePublishContent.js",

packages/legend-extension-dsl-diagram/_package.config.js

@@ -14,20 +14,11 @@
  * limitations under the License.
  */
 
-import { generateBundleCopyrightText } from '../../scripts/copyright/PackageCopyrightHelper.js';
-import { dirname } from 'path';
-import { fileURLToPath } from 'url';
-
-const __dirname = dirname(fileURLToPath(import.meta.url));
-
 export default {
   publish: {
     typescript: {
       main: './tsconfig.build.json',
       others: ['./tsconfig.package.json'],
     },
   },
-  build: {
-    copyrightText: generateBundleCopyrightText(__dirname),
-  },
 };

packages/legend-extension-dsl-diagram/package.json

@@ -25,11 +25,11 @@
   "types": "lib/index.d.ts",
   "scripts": {
     "build": "yarn clean && yarn build:sass && yarn build:tsc",
-    "build:sass": "cross-env INIT_CWD=$INIT_CWD sass ./style/index.scss ./lib/index.css --style=compressed && node ../../scripts/copyright/addBundledCodeCopyrightHeader.js ./lib/index.css",
+    "build:sass": "cross-env INIT_CWD=$INIT_CWD node ../../scripts/workflow/buildSass.js",
     "build:tsc": "tsc --project ./tsconfig.build.json",
     "clean": "rimraf \"lib\" \"build\"",
     "dev": "npm-run-all --parallel dev:sass dev:tsc",
-    "dev:sass": "sass ./style/index.scss ./lib/index.css --watch",
+    "dev:sass": "sass style lib --watch",
     "dev:tsc": "tsc --watch --preserveWatchOutput",
     "lint:js": "cross-env NODE_ENV=production eslint --cache --cache-location ./build/.eslintcache --report-unused-disable-directives --parser-options=project:\"./tsconfig.json\" \"./src/**/*.{js,ts,tsx}\"",
     "publish:prepare": "node ../../scripts/release/preparePublishContent.js",