diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e9f573397b..c6c3644ba9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -3,6 +3,9 @@ # Please see the documentation for all configuration options: # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates +# TODO: update for pnpm when possible: +# https://github.com/dependabot/dependabot-core/issues/1736 + version: 2 updates: # Root package.json diff --git a/.github/workflows/node-js-ci.yml b/.github/workflows/node-js-ci.yml index 1bb801f76b..8c1becc1ce 100644 --- a/.github/workflows/node-js-ci.yml +++ b/.github/workflows/node-js-ci.yml @@ -16,10 +16,22 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2.1.5 - - name: Install dependencies and run prettier-check + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.0.2 + run_install: true + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 + env: + cache-name: cache-pnpm-modules + with: + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- + - name: Run prettier-check run: | - npm install - npm run prettier-check + pnpm run prettier-check # Make sure eslint passes eslint: @@ -28,10 +40,22 @@ jobs: steps: - uses: actions/checkout@v2 - uses: actions/setup-node@v2.1.5 - - name: Install dependencies and run eslint + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.0.2 + run_install: true + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 + env: + cache-name: cache-pnpm-modules + with: + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- + - name: Run eslint run: | - npm install - npm run eslint + pnpm run eslint # Run unit tests on all platforms/versions of node unit: @@ -43,25 +67,25 @@ jobs: node-version: [12.x, 14.x] steps: - uses: actions/checkout@v2 - - uses: actions/cache@v2.1.5 + - uses: actions/setup-node@v2.1.5 + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.0.2 + run_install: true + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 env: - cache-name: cache-node-modules + cache-name: cache-pnpm-modules with: - # npm cache files are stored in `~/.npm` on Linux/macOS - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package.json') }} + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- - - uses: actions/setup-node@v2.1.5 - with: - node-version: ${{ matrix.node-version }} + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v1 - - name: Install dependencies and run tests with default env + - name: Run unit tests with default env run: | - npm install + # pnpm can't handle passing args through to run scripts: https://github.com/pnpm/pnpm/issues/2764 npm run jest --max-workers ${{ steps.cpu-cores.outputs.count }} # Run end-to-end tests along with the microservices in docker-compose on Linux @@ -72,19 +96,26 @@ jobs: matrix: node-version: [12.x, 14.x] steps: - - uses: actions/cache@v2.1.5 + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2.1.5 + - uses: pnpm/action-setup@v2.0.1 + with: + version: 6.0.2 + run_install: true + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 + env: + cache-name: cache-pnpm-modules with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} restore-keys: | - ${{ runner.os }}-buildx- - - uses: actions/checkout@v2 + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- - uses: actions/setup-node@v2.1.5 with: node-version: ${{ matrix.node-version }} - uses: microsoft/playwright-github-action@v1 - - name: Start microservices and run tests + - name: Start microservices and run e2e tests run: | - npm install - npm run services:start - npm run jest:e2e + pnpm run services:start + pnpm run jest:e2e diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 98e0312bcd..b9b061a2f9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -15,23 +15,48 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 + env: + cache-name: cache-pnpm-modules + with: + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- - uses: actions/setup-node@v2.1.5 - - name: Install dependencies and run prettier-check + - uses: pnpm/action-setup@v1.2.1 + with: + version: 6.0.1 + run_install: true + - name: Run prettier-check run: | - npm install - npm run prettier-check + pnpm run prettier-check + # Make sure eslint passes # Make sure eslint passes eslint: name: ESLint Check runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 + env: + cache-name: cache-pnpm-modules + with: + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} + restore-keys: | + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- - uses: actions/setup-node@v2.1.5 - - name: Install dependencies and run eslint + - uses: pnpm/action-setup@v1.2.1 + with: + version: 6.0.1 + run_install: true + - name: Run eslint run: | - npm install - npm run eslint + pnpm run eslint # Run unit tests on all platforms/versions of node unit: @@ -43,26 +68,27 @@ jobs: node-version: [12.x, 14.x] steps: - uses: actions/checkout@v2 - - uses: actions/cache@v2.1.5 + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 env: - cache-name: cache-node-modules + cache-name: cache-pnpm-modules with: - # npm cache files are stored in `~/.npm` on Linux/macOS - path: ~/.npm - key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package.json') }} + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} restore-keys: | - ${{ runner.os }}-build-${{ env.cache-name }}- - ${{ runner.os }}-build- - ${{ runner.os }}- + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- - uses: actions/setup-node@v2.1.5 with: node-version: ${{ matrix.node-version }} + - uses: pnpm/action-setup@v1.2.1 + with: + version: 6.0.1 + run_install: true - name: Get number of CPU cores uses: SimenB/github-actions-cpu-cores@v1 - - name: Install dependencies and run tests with default env + - name: Run unit tests with default env run: | - npm install - npm run jest --max-workers ${{ steps.cpu-cores.outputs.count }} + pnpm run jest --max-workers ${{ steps.cpu-cores.outputs.count }} # Run end-to-end tests along with the microservices in docker-compose on Linux e2e: @@ -72,22 +98,25 @@ jobs: matrix: node-version: [12.x, 14.x] steps: - - uses: actions/cache@v2.1.5 + - uses: actions/checkout@v2 + - name: Cache pnpm modules + uses: actions/cache@v2.1.5 + env: + cache-name: cache-pnpm-modules with: - path: /tmp/.buildx-cache - key: ${{ runner.os }}-buildx-${{ github.sha }} + path: ~/.pnpm-store + key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}-${{ hashFiles('**/package.json') }} restore-keys: | - ${{ runner.os }}-buildx- - - uses: actions/checkout@v2 + ${{ runner.os }}-build-${{ env.cache-name }}-${{ matrix.node-version }}- - uses: actions/setup-node@v2.1.5 with: node-version: ${{ matrix.node-version }} - uses: microsoft/playwright-github-action@v1 - - name: Start microservices and run tests + - name: Start microservices and run e2e tests run: | - npm install - npm run services:start - npm run jest:e2e + pnpm run services:start + pnpm run jest:e2e + release: needs: [prettier, eslint, unit, e2e] runs-on: ubuntu-latest diff --git a/.husky/pre-commit b/.husky/pre-commit index d0612ad370..a9e30b9bec 100755 --- a/.husky/pre-commit +++ b/.husky/pre-commit @@ -1,4 +1,4 @@ #!/bin/sh . "$(dirname "$0")/_/husky.sh" -npm run pre-commit +pnpm run pre-commit diff --git a/.npmrc b/.npmrc index aaab65b7ac..43424afb6c 100644 --- a/.npmrc +++ b/.npmrc @@ -1,13 +1,5 @@ -# Don't create or read package-lock.json -package-lock=false -# Use exact versions of dependencies vs. semver minor updates -save-exact=true - # Allow running postinstall script in Docker unsafe-perm=true # Set empty tag version prefix tag-version-prefix="" - -#Set legacy peer dependencies to 'true', (needed for npm7.x compatibility for now) -legacy-peer-deps=true diff --git a/docs/environment-setup.md b/docs/environment-setup.md index 3ac7416c78..a2de065ed1 100644 --- a/docs/environment-setup.md +++ b/docs/environment-setup.md @@ -2,7 +2,8 @@ ### Prerequisites: -- [Node.js (npm)](https://nodejs.org/en/download/) +- [Node.js](https://nodejs.org/en/download/) +- [pnpm](https://pnpm.io/) (we use `pnpm` vs. `npm` for package installation) - [Redis](https://redis.io/) (2 methods) - Use [Docker and docker-compose](https://docs.docker.com/install/) - Install as a [native application](#Install-Redis-as-a-native-application) @@ -137,6 +138,32 @@ _NOTE: This will not work on WSL (Windows Subsystem for Linux). Use the approach ## After installing the prerequisites: +### Installing Dependencies with pnpm + +Telescope is structured as a [monrepo](https://en.wikipedia.org/wiki/Monorepo) with more than a dozen separate projects all contained within a single git repository. Because monorepos often share many of the same dependencies, there can be a lot of duplication (and waste) when installing packages. + +To help reduce the number of packages that must be stored in `node_modules`, we use [pnpm](https://pnpm.io/) intead of the usual `npm` command. With [pnpm we can save a lot of disk space and installation time](https://pnpm.io/motivation). + +The cost of using pnpm is that there are a few differences in how [packages are installed](https://pnpm.io/cli/install), [added to package.json](https://pnpm.io/cli/add), etc. The [docs](https://pnpm.io/pnpm-cli) have full details on how it works, and after a few minutes you'll have no trouble. + +To install all of Telescope's dependencies, use the following command: + +```bash +pnpm install +``` + +Similarly, to run tests: + +```bash +pnpm test +``` + +And to run npm scripts: + +```bash +pnpm run +``` + ### Start Docker ```bash @@ -162,9 +189,9 @@ _Note: Make sure you're running these commands in the root of telescope project. This is the default setting, you do not need to copy or modify any `env` file. ```bash -npm run services:start +pnpm run services:start -npm run dev +pnpm run dev ``` Microservices will start downloading feeds and processing them until stopped. For more information about the services, please read [Telescope API Services](../src/api/readme.md). @@ -176,7 +203,7 @@ If this doesn't work for you, it is possible that you have an old `.env` file in ```bash cp config/env.staging .env -npm run dev +pnpm run dev ``` This will provide you staging back-end without running it locally. @@ -190,22 +217,22 @@ This one depends on which part you're working with. For example, if you want to After modify the `.env` file, run these commands, ```bash -npm run services:start +pnpm run services:start -npm run dev +pnpm run dev ``` #### Want to run `auth/image/posts` service solely -`npm run services:start auth` or `npm run services:start image` or `npm run services:start posts` +`pnpm run services:start auth` or `pnpm run services:start image` or `pnpm run services:start posts` #### Want to update the Docker image(s) after making some changes Run the following commands to rebuild the image(s): ```bash -npm run services:clean -npm run services:start +pnpm run services:clean +pnpm run services:start ``` #### Login/SSO: diff --git a/docs/git-workflow.md b/docs/git-workflow.md index b69b9e7135..b486a01c11 100644 --- a/docs/git-workflow.md +++ b/docs/git-workflow.md @@ -14,7 +14,7 @@ When working on fixing bugs, please use the following workflow: git checkout master git pull upstream master ``` -1. Additionally, it is a good idea to run `npm install` to make sure everything is up to date and you have everything necessary. +1. Additionally, it is a good idea to run `pnpm install` to make sure everything is up to date and you have everything necessary. 1. Create a branch for your work, using the issue number: ``` git checkout -b issue-123 @@ -28,13 +28,13 @@ When working on fixing bugs, please use the following workflow: ## Testing Your Code -Run the test suite, using `npm test`. Fix any lint errors, warnings, or other failures (NOTE: if you're not sure what an eslint rule means, [look it up in the docs](https://eslint.org/docs/rules/)): +Run the test suite, using `pnpm test`. Fix any lint errors, warnings, or other failures (NOTE: if you're not sure what an eslint rule means, [look it up in the docs](https://eslint.org/docs/rules/)): ``` -npm test +pnpm test ...if there are lint errors, try having eslint fix them for you -npm run eslint-fix -npm test +pnpm run eslint-fix +pnpm test ...manually fix any errors yourself, rerunning npm test each time to confirm ``` @@ -42,26 +42,16 @@ You can also run the tests in _watch_ mode, so that they will automatically ru-r when you make changes: ``` -npm run jest-watch +pnpm run jest-watch ``` In addition, you can run individual tests, in both normal or watch mode, by adding the name of a file. For example, to run tests in a file called parser.test.js: ``` -npm test parser +pnpm test parser ``` -You can add feeds to the queue manually using `add-feed` followed by the name of the blogger and url of the feed. This can be useful for testing purposes. - -First, add a link to the binary: - -`npm link` - -Then use `add-feed` - -`add-feed --name "Bender Bending Rodriguez" --url futurama.wordpress.com/feed` - ## Debugging The Telescope backend can be debugged using VSCode. See the [VSCode Debugging docs](https://code.visualstudio.com/docs/editor/debugging#_launch-configurations) and use the Launch Telescope configuration to start the server and debugger within VSCode. @@ -110,7 +100,7 @@ fixup b85d7a9 Final Build ...edit files to fix review comments git add file1 git commit -m "Updated file1 to fix review comments" - npm test + pnpm test ...if the tests fail, fix things, and repeat until they pass git push origin issue-123 ``` @@ -124,7 +114,7 @@ fixup b85d7a9 Final Build git add file1 git rebase --continue ...repeat until your rebase completes. If you get stuck, use git rebase --abort to stop - npm install + pnpm install ...this is likely necessary to update your node_modules/, see below git push origin issue-123 -f ``` @@ -135,6 +125,5 @@ If you get stuck with any of this, ask in your issue or pull request, and we'll If you are doing any work that relates to the `package.json` file, you need to do this with care. Here are some tips: -- we don't include `package-lock.json` in our tree. Instead, we use [exact](https://docs.npmjs.com/misc/config#save-exact) versions in `package.json`. When you `npm install` a package, we specify an exact version number vs. using a semver range. -- don't hand-edit this file to add packages. Instead, use `npm install --save package-name` or `npm install --save-dev package-name` to add packages to the `dependencies` or `devDependencies` sections. -- if you touch `package.json`, always re-run `npm install`. +- don't hand-edit any of the `package.json` files, use [pnpm add](https://pnpm.io/cli/add) instead. +- we store the `pnpm-lock.yaml` lock file in git. See https://pnpm.io/git#lockfiles for more details. If you add or update dependencies, you will need to also include the updated lockfile. diff --git a/docs/reports.md b/docs/reports.md index 92a34a6134..75e856d718 100644 --- a/docs/reports.md +++ b/docs/reports.md @@ -8,7 +8,7 @@ The report you can generate is test coverage information. To generate this repor run the following command: ``` -npm run coverage +pnpm run coverage ``` After the tests complete, a text summary report is printed. However, a much more diff --git a/package.json b/package.json index 6b059e181c..a29c7e7017 100644 --- a/package.json +++ b/package.json @@ -10,26 +10,15 @@ "scripts": { "api:start": "docker-compose -f ./src/api/docker-compose-api.yml up --build -d", "api:stop": "docker-compose -f ./src/api/docker-compose-api.yml down", - "install:auth-service": "cd src/api/auth && npm install", - "install:image-service": "cd src/api/image && npm install", - "install:users-service": "cd src/api/users && npm install", - "install:posts-service": "cd src/api/posts && npm install", - "install:search-service": "cd src/api/search && npm install", - "install:feed-discovery-service": "cd src/api/feed-discovery && npm install", - "install:parser-service": "cd src/api/parser && npm install", - "install:autodeployment": "cd tools/autodeployment && npm install", - "install:next": "cd src/web && npm install", - "build": "npm run build --prefix src/web --", - "dev": "npm run dev --prefix src/web --", - "develop": "npm run dev", + "build": "pnpm --dir src/web build --", + "dev": "pnpm --dir src/web dev --", + "develop": "pnpm run dev", "eslint": "eslint --config .eslintrc.js", "eslint-fix": "eslint --config .eslintrc.js --fix", - "lint": "npm run eslint", - "postinstall": "run-s install:*", + "lint": "pnpm run eslint", "prettier": "prettier --write \"./**/*.{md,jsx,json,html,css,js,yml}\"", "prettier-check": "prettier --check \"./**/*.{md,jsx,json,html,css,js,yml}\"", - "pretest": "npm run lint", - "test": "npm run jest", + "test": "run-s lint jest", "jest": "cross-env NODE_ENV=test LOG_LEVEL=error MOCK_REDIS=1 FEED_URL_INTERVAL_MS=200 jest -c jest.config.js --", "jest:e2e": "jest -c jest.config.e2e.js --", "coverage": "cross-env NODE_ENV=test LOG_LEVEL=silent MOCK_REDIS=1 FEED_URL_INTERVAL_MS=200 jest -c jest.config.js --collectCoverage --", @@ -45,7 +34,8 @@ "services:stop": "node bin/services-stop.js", "services:logs": "node bin/services-logs.js --", "services:clean": "node bin/services-clean.js", - "prepare": "husky install" + "prepare": "husky install", + "preinstall": "npx only-allow pnpm" }, "repository": "Seneca-CDOT/telescope", "license": "BSD-2-Clause", @@ -133,6 +123,7 @@ "ts-jest": "26.5.4" }, "engines": { - "node": ">=12.0.0" + "node": ">=12.0.0", + "pnpm": ">=6" } } diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml new file mode 100644 index 0000000000..158ba66626 --- /dev/null +++ b/pnpm-workspace.yaml @@ -0,0 +1,6 @@ +# pnpm workspace definition: https://pnpm.io/pnpm-workspace_yaml +packages: + # web front-end + - 'src/web' + # microservices + - 'src/api/**' diff --git a/src/api/readme.md b/src/api/readme.md index de95877078..ab7d96b35a 100644 --- a/src/api/readme.md +++ b/src/api/readme.md @@ -30,13 +30,19 @@ docker-compose --env-file config/env.development up -d You can access logs for one or more services: ``` -npm run services:logs image firebase +pnpm run services:logs image firebase ``` To stop the services: ``` -npm run services:stop +pnpm run services:stop +``` + +To see logs for one of the running services (e.g., `image`): + +``` +pnpm run services:logs image ``` ## API Lookup Table diff --git a/test/test-web-content/manual-auth/index.html b/test/test-web-content/manual-auth/index.html index 6e340a08fb..038c238856 100644 --- a/test/test-web-content/manual-auth/index.html +++ b/test/test-web-content/manual-auth/index.html @@ -20,7 +20,7 @@

Telescope Auth Service Test

-        npm run services:start
+        pnpm run services:start