From e72b33ad468af024a7187be671d5e39ed4b71df2 Mon Sep 17 00:00:00 2001 From: tmair Date: Mon, 27 Mar 2023 11:49:08 +0200 Subject: [PATCH] docs: improve search for documentation (#6952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * docs: update search to aio search * chore: use custom stop word filter for lunr * docs: applay review suggestions * docs: remove stop word filtering during search * docs: improve first search query to search only in title * docs: remove deprecated import paths from search results * chore: commit package-lock.json --------- Co-authored-by: Mladen Jakovljević (cherry picked from commit 0175187f8b56ab5988a629c52cf93f9deedfb14e) --- docs_app/.gitignore | 3 - docs_app/angular.json | 1 + docs_app/package-lock.json | 369 +++------ docs_app/package.json | 4 +- docs_app/src/app/app.component.ts | 2 +- docs_app/src/app/search/search-worker.js | 103 --- docs_app/src/app/search/search.service.ts | 25 +- docs_app/src/app/search/search.worker.ts | 174 +++++ docs_app/src/app/shared/web-worker-message.ts | 5 + docs_app/src/app/shared/web-worker.ts | 26 +- .../angular-base-package/ignore-words.json | 703 ++++++++++++++++++ .../angular-base-package/ignore.words | 701 ----------------- .../transforms/angular-base-package/index.js | 23 +- .../processors/generateKeywords.js | 245 +++--- .../processors/generateKeywords.spec.js | 338 ++++++--- docs_app/tsconfig.worker.json | 16 + 16 files changed, 1441 insertions(+), 1297 deletions(-) delete mode 100644 docs_app/src/app/search/search-worker.js create mode 100644 docs_app/src/app/search/search.worker.ts create mode 100644 docs_app/src/app/shared/web-worker-message.ts create mode 100644 docs_app/tools/transforms/angular-base-package/ignore-words.json delete mode 100644 docs_app/tools/transforms/angular-base-package/ignore.words create mode 100644 docs_app/tsconfig.worker.json diff --git a/docs_app/.gitignore b/docs_app/.gitignore index a297f88f3a..10934931a7 100644 --- a/docs_app/.gitignore +++ b/docs_app/.gitignore @@ -46,8 +46,5 @@ protractor-results*.txt .DS_Store Thumbs.db -# copied dependencies -src/assets/js/lunr* - assets/images/svgs/* !assets/images/svgs/.gitkeep \ No newline at end of file diff --git a/docs_app/angular.json b/docs_app/angular.json index 64b5fd5000..569aefad19 100644 --- a/docs_app/angular.json +++ b/docs_app/angular.json @@ -15,6 +15,7 @@ "index": "src/index.html", "main": "src/main.ts", "tsConfig": "src/tsconfig.app.json", + "webWorkerTsConfig": "tsconfig.worker.json", "namedChunks": true, "polyfills": "src/polyfills.ts", "assets": [ diff --git a/docs_app/package-lock.json b/docs_app/package-lock.json index 7dc02a5a5d..859eddc5d1 100644 --- a/docs_app/package-lock.json +++ b/docs_app/package-lock.json @@ -7,7 +7,6 @@ "": { "name": "rxjs.dev", "version": "1.0.0", - "hasInstallScript": true, "license": "MIT", "dependencies": { "@angular/animations": "^13.1.1", @@ -34,11 +33,12 @@ "@angular/cli": "^13.1.2", "@angular/compiler-cli": "^13.1.1", "@jsdevtools/rehype-inline-svg": "^1.1.1", - "@swirly/parser": "^0.17.6", - "@swirly/renderer-node": "^0.17.6", - "@swirly/types": "^0.17.6", + "@swirly/parser": "^0.18.1", + "@swirly/renderer-node": "^0.18.2", + "@swirly/types": "^0.18.1", "@types/jasmine": "~3.6.0", "@types/jasminewd2": "^2.0.3", + "@types/lunr": "^2.3.3", "@types/node": "^12.11.1", "@types/svgo": "^1.3.3", "archiver": "^3.0.0", @@ -51,7 +51,6 @@ "css-selector-parser": "^1.3.0", "dgeni": "^0.4.14", "dgeni-packages": "^0.30.0", - "entities": "^1.1.1", "eslint": "^5.16.0", "eslint-plugin-jasmine": "^2.2.0", "firebase-tools": "^9.3.0", @@ -85,16 +84,13 @@ "rimraf": "^2.6.1", "semver": "^6.1.1", "shelljs": "^0.8.3", + "stemmer": "^2.0.0", "svgo": "^1.3.2", "svgson": "^4.1.0", - "swirly-parser": "^0.13.6", - "swirly-renderer-node": "^0.13.6", - "swirly-types": "^0.13.6", "tree-kill": "^1.2.2", "ts-node": "^8.2.0", "tslint": "~6.1.0", "typescript": "4.5.4", - "uglify-js": "^3.6.0", "unist-util-filter": "^1.0.2", "unist-util-source": "^1.0.5", "unist-util-visit": "^1.4.1", @@ -3721,15 +3717,15 @@ "integrity": "sha512-4fCa1YGdGXiWsCEsFQN+O6RP4JibmEMO58S9wzbFSko6lu1zLqkTBomRt1LCF7TImWeJaSSMDMqin58bey+N9Q==" }, "node_modules/@swirly/parser": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/parser/-/parser-0.17.6.tgz", - "integrity": "sha512-Z6OrvKwmKsbxPbOc4fmSLowuho9Tt7m7SGsG4tkEIls61zvOb0P+XEvcTX3fEtUtuDL5CLMDRqfy4fV/J73SSQ==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/parser/-/parser-0.18.1.tgz", + "integrity": "sha512-WET7DtMma+j9DYA5u2h5dEWz8oZOnvsrGxYO6RWzyK2ZrRjHiqTwTHbiWwUUaKQVn3CGZMSlJxn+6RDORrLTDg==", "dev": true, "dependencies": { - "@swirly/parser-rxjs": "^0.17.6" + "@swirly/parser-rxjs": "^0.18.1" }, "engines": { - "node": ">=10" + "node": ">=14" }, "funding": { "type": "github", @@ -3737,12 +3733,12 @@ } }, "node_modules/@swirly/parser-rxjs": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/parser-rxjs/-/parser-rxjs-0.17.6.tgz", - "integrity": "sha512-cmJdSFokjNQHrdJjvs1oXwsNQfMbDeYdkq8WmHFq5AbK1kM5MT/RrDnyoRm6CLqrcqO6b/CBlJqjDIaPXL7KIg==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/parser-rxjs/-/parser-rxjs-0.18.1.tgz", + "integrity": "sha512-DBBrw/9DWgkuSvCPXJ8NjUcySOMS0T41MaMg6EZDigdbQXn7ktWlYN2n1M6CBjkDqpAM20/B0eJqVyCZll+rhw==", "dev": true, "engines": { - "node": ">=10" + "node": ">=14" }, "funding": { "type": "github", @@ -3750,16 +3746,16 @@ } }, "node_modules/@swirly/renderer": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/renderer/-/renderer-0.17.6.tgz", - "integrity": "sha512-pp4g9VevkYUPh+wwuOSPPblVHLtw3UcoF/OfAG0BbaC6DcHLsDEOk2CCZuidUb3ozvwo1fdHZKeWbs+ceXckOg==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/renderer/-/renderer-0.18.1.tgz", + "integrity": "sha512-48zG4S6qZja2sHihKbLt48BsZMELbWUHUrKIhjjdNs/HXaV+Vg65/75h9kDg69mq4aDLFOxGx9aAdgSqtAHArQ==", "dev": true, "dependencies": { - "@swirly/theme-default-light": "^0.17.6", + "@swirly/theme-default-light": "^0.18.1", "simple-sha1": "^3.1.0" }, "engines": { - "node": ">=10" + "node": ">=14" }, "funding": { "type": "github", @@ -3767,16 +3763,16 @@ } }, "node_modules/@swirly/renderer-node": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/renderer-node/-/renderer-node-0.17.6.tgz", - "integrity": "sha512-eUkgp/zB5k7HaUO1T2KLyfihZpCTgJ3h9clnNGNUqL/lewqe2v1q/oqLXDJKhfSsefH9C+x/J2+hNFKo2kmYMg==", + "version": "0.18.2", + "resolved": "https://registry.npmjs.org/@swirly/renderer-node/-/renderer-node-0.18.2.tgz", + "integrity": "sha512-Qcvqzjd6cEac/PZOv3mUAb2myEAnZbxCVb45kkjLRcs38Nqj4pq2aZH1E8E4OYR3XeZmHh8S7EtWO6k7phnqQg==", "dev": true, "dependencies": { - "@swirly/renderer": "^0.17.6", - "xmldom": "^0.5.0" + "@swirly/renderer": "^0.18.1", + "@xmldom/xmldom": "^0.8.2" }, "engines": { - "node": ">=10" + "node": ">=14" }, "funding": { "type": "github", @@ -3784,12 +3780,12 @@ } }, "node_modules/@swirly/theme-default-base": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/theme-default-base/-/theme-default-base-0.17.6.tgz", - "integrity": "sha512-HbFMyL9xZjIzDhWzhAHoU7slfkET2euyLCB7lzOlRprPzfjatMTmCL30ji+Z6wRdQEITsrERCwiy5hfyo+rQEQ==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/theme-default-base/-/theme-default-base-0.18.1.tgz", + "integrity": "sha512-lX6ThJxHIsBj3So4BsVRanMf8Cw/1TupLUC9xeArGXzwLKCKAQXica8piDJd+spEU78A2r+g6AHgwqa4JuKZ6Q==", "dev": true, "engines": { - "node": ">=10" + "node": ">=14" }, "funding": { "type": "github", @@ -3797,15 +3793,15 @@ } }, "node_modules/@swirly/theme-default-light": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/theme-default-light/-/theme-default-light-0.17.6.tgz", - "integrity": "sha512-KS6CmOcBqAVkJpx+K0rsNSSKGONrWsz7o7xT6Z4gOw0IFGwc/6/xjxvXGrMttbqutkGVv1vZEbl+wrAGDQ84WA==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/theme-default-light/-/theme-default-light-0.18.1.tgz", + "integrity": "sha512-PpjBvZykHOxLz1I+lTxfShvM3PUZBjD1bfnjArEP8mKu4Ng6ltQ/CZauQQDuvRvpC6CtnzBK1Gg/BNAjP93k0A==", "dev": true, "dependencies": { - "@swirly/theme-default-base": "^0.17.6" + "@swirly/theme-default-base": "^0.18.1" }, "engines": { - "node": ">=10" + "node": ">=14" }, "funding": { "type": "github", @@ -3813,12 +3809,12 @@ } }, "node_modules/@swirly/types": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/types/-/types-0.17.6.tgz", - "integrity": "sha512-BHAEL95TTC1Trn2xHl5zJFyPDiDdGSo2GM32SLu1KC000vunjFPRvOVnfA8f9WflJz/It46ZiMaXl4a6g8F9ag==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/types/-/types-0.18.1.tgz", + "integrity": "sha512-TlXBe3nSfF5vuODhfP8ZKzmnqflWEZ7nXS6WQoS9loCOphMLBjycsm/01sM3SXrszCvbqRWFJ7nZrtSLQcgyMQ==", "dev": true, "engines": { - "node": ">=10" + "node": ">=14" }, "funding": { "type": "github", @@ -3963,6 +3959,12 @@ "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==", "dev": true }, + "node_modules/@types/lunr": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@types/lunr/-/lunr-2.3.4.tgz", + "integrity": "sha512-j4x4XJwZvorEUbA519VdQ5b9AOU9TSvfi8tvxMAfP8XzNLtFex7A8vFQwqOx3WACbV0KMXbACV3cZl4/gynQ7g==", + "dev": true + }, "node_modules/@types/mdast": { "version": "3.0.10", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.10.tgz", @@ -4198,6 +4200,15 @@ "resolved": "https://registry.npmjs.org/@webcomponents/custom-elements/-/custom-elements-1.5.0.tgz", "integrity": "sha512-c+7jPQCs9h/BYVcZ2Kna/3tsl3A/9EyXfvWjp5RiTDm1OpTcbZaCa1z4RNcTe/hUtXaqn64JjNW1yrWT+rZ8gg==" }, + "node_modules/@xmldom/xmldom": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.2.tgz", + "integrity": "sha512-+R0juSseERyoPvnBQ/cZih6bpF7IpCXlWbHRoCRzYzqpz6gWHOgf8o4MOEf6KBVuOyqU+gCNLkCWVIJAro8XyQ==", + "dev": true, + "engines": { + "node": ">=10.0.0" + } + }, "node_modules/@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", @@ -23320,6 +23331,19 @@ "node": ">=0.10.0" } }, + "node_modules/stemmer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/stemmer/-/stemmer-2.0.1.tgz", + "integrity": "sha512-bkWvSX2JR4nSZFfs113kd4C6X13bBBrg4fBKv2pVdzpdQI2LA5pZcWzTFNdkYsiUNl13E4EzymSRjZ0D55jBYg==", + "dev": true, + "bin": { + "stemmer": "cli.js" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", @@ -23769,100 +23793,6 @@ "upper-case": "^1.1.1" } }, - "node_modules/swirly-parser": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-parser/-/swirly-parser-0.13.10.tgz", - "integrity": "sha512-EJvfYAHI10WDL60mpltu/lpNHtY77Elj2waAr8LPDwKreohk9B9XJ+m8V33i0iIKLsmDksIFl4l0RRVAQx7t7g==", - "deprecated": "please use @swirly/parser instead", - "dev": true, - "dependencies": { - "swirly-parser-rxjs": "^0.13.10" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/swirly-parser-rxjs": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-parser-rxjs/-/swirly-parser-rxjs-0.13.10.tgz", - "integrity": "sha512-6wmdclC0zscSWjIKPgXkO5JurM/mVtV2tmzr1+j2ZEKiB4gKLV0QKSb4qGu87GtVYuzFjN4H4aCDH99Suq2GMA==", - "deprecated": "please use @swirly/parser-rxjs instead", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/swirly-renderer": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-renderer/-/swirly-renderer-0.13.10.tgz", - "integrity": "sha512-RSgdb9wE1SKcae49O8UjYaU8ayZWv6u/zRfkARGbuouYj2rXLceHyKAHBu9OAg4tRkyKZ1ioyxKFenHNeQE3rA==", - "deprecated": "please use @swirly/renderer instead", - "dev": true, - "dependencies": { - "simple-sha1": "^3.0.1", - "swirly-theme-default-light": "^0.13.10" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/swirly-renderer-node": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-renderer-node/-/swirly-renderer-node-0.13.10.tgz", - "integrity": "sha512-YL9umzpv8Nl9xAwxr1z+0CTqo94TcQFgRbzr/RJL1t6RG+b51q0x/O0oKG/jxIH9hnr6H3PJbzJwNrrwfHVsrQ==", - "deprecated": "please use @swirly/renderer-node instead", - "dev": true, - "dependencies": { - "swirly-renderer": "^0.13.10", - "xmldom": "^0.3.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/swirly-renderer-node/node_modules/xmldom": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.3.0.tgz", - "integrity": "sha512-z9s6k3wxE+aZHgXYxSTpGDo7BYOUfJsIRyoZiX6HTjwpwfS2wpQBQKa2fD+ShLyPkqDYo5ud7KitmLZ2Cd6r0g==", - "deprecated": "Deprecated due to CVE-2021-21366 resolved in 0.5.0", - "dev": true, - "engines": { - "node": ">=10.0.0" - } - }, - "node_modules/swirly-theme-default-base": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-theme-default-base/-/swirly-theme-default-base-0.13.10.tgz", - "integrity": "sha512-s1g9llE+fVdHemufzXcAM2p95OLCuwQ6/AB8tSAD9CStr0mZXWd6lPVGNw0i7T77bRe7TZirs/OeOq5gr2VgIw==", - "deprecated": "please use @swirly/theme-default-base instead", - "dev": true, - "engines": { - "node": ">=10" - } - }, - "node_modules/swirly-theme-default-light": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-theme-default-light/-/swirly-theme-default-light-0.13.10.tgz", - "integrity": "sha512-2EXGLtcFKJHWN8snOQXqon0d3h2tS6Pu4v2TnLp7GdYU2ggEzqEKCfHWm5uEuW/j/HxBevlOBbdU0bZA1YRxJg==", - "deprecated": "please use @swirly/theme-default-light instead", - "dev": true, - "dependencies": { - "swirly-theme-default-base": "^0.13.10" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/swirly-types": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-types/-/swirly-types-0.13.10.tgz", - "integrity": "sha512-omsVUaHqCTcJV+kPt2iebg2XA4al2109MG6wkS/R2tBH1U1mv0TIaPeBSfbCctrt3EJKbazkJjY0axe1Fb1m1A==", - "deprecated": "please use @swirly/types instead", - "dev": true, - "engines": { - "node": ">=10" - } - }, "node_modules/symbol-observable": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-4.0.0.tgz", @@ -24617,18 +24547,6 @@ "node": "*" } }, - "node_modules/uglify-js": { - "version": "3.13.2", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.13.2.tgz", - "integrity": "sha512-SbMu4D2Vo95LMC/MetNaso1194M1htEA+JrqE9Hk+G2DhI+itfS9TRu9ZKeCahLDNa/J3n4MqUJ/fOHMzQpRWw==", - "dev": true, - "bin": { - "uglifyjs": "bin/uglifyjs" - }, - "engines": { - "node": ">=0.8.0" - } - }, "node_modules/ultron": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", @@ -26299,15 +26217,6 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, - "node_modules/xmldom": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.5.0.tgz", - "integrity": "sha512-Foaj5FXVzgn7xFzsKeNIde9g6aFBxTPi37iwsno8QvApmtg7KYrr+OPyRHcJF7dud2a5nGRBXK3n0dL62Gf7PA==", - "dev": true, - "engines": { - "node": ">=10.0.0" - } - }, "node_modules/xregexp": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.4.1.tgz", @@ -29103,59 +29012,59 @@ "integrity": "sha512-4fCa1YGdGXiWsCEsFQN+O6RP4JibmEMO58S9wzbFSko6lu1zLqkTBomRt1LCF7TImWeJaSSMDMqin58bey+N9Q==" }, "@swirly/parser": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/parser/-/parser-0.17.6.tgz", - "integrity": "sha512-Z6OrvKwmKsbxPbOc4fmSLowuho9Tt7m7SGsG4tkEIls61zvOb0P+XEvcTX3fEtUtuDL5CLMDRqfy4fV/J73SSQ==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/parser/-/parser-0.18.1.tgz", + "integrity": "sha512-WET7DtMma+j9DYA5u2h5dEWz8oZOnvsrGxYO6RWzyK2ZrRjHiqTwTHbiWwUUaKQVn3CGZMSlJxn+6RDORrLTDg==", "dev": true, "requires": { - "@swirly/parser-rxjs": "^0.17.6" + "@swirly/parser-rxjs": "^0.18.1" } }, "@swirly/parser-rxjs": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/parser-rxjs/-/parser-rxjs-0.17.6.tgz", - "integrity": "sha512-cmJdSFokjNQHrdJjvs1oXwsNQfMbDeYdkq8WmHFq5AbK1kM5MT/RrDnyoRm6CLqrcqO6b/CBlJqjDIaPXL7KIg==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/parser-rxjs/-/parser-rxjs-0.18.1.tgz", + "integrity": "sha512-DBBrw/9DWgkuSvCPXJ8NjUcySOMS0T41MaMg6EZDigdbQXn7ktWlYN2n1M6CBjkDqpAM20/B0eJqVyCZll+rhw==", "dev": true }, "@swirly/renderer": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/renderer/-/renderer-0.17.6.tgz", - "integrity": "sha512-pp4g9VevkYUPh+wwuOSPPblVHLtw3UcoF/OfAG0BbaC6DcHLsDEOk2CCZuidUb3ozvwo1fdHZKeWbs+ceXckOg==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/renderer/-/renderer-0.18.1.tgz", + "integrity": "sha512-48zG4S6qZja2sHihKbLt48BsZMELbWUHUrKIhjjdNs/HXaV+Vg65/75h9kDg69mq4aDLFOxGx9aAdgSqtAHArQ==", "dev": true, "requires": { - "@swirly/theme-default-light": "^0.17.6", + "@swirly/theme-default-light": "^0.18.1", "simple-sha1": "^3.1.0" } }, "@swirly/renderer-node": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/renderer-node/-/renderer-node-0.17.6.tgz", - "integrity": "sha512-eUkgp/zB5k7HaUO1T2KLyfihZpCTgJ3h9clnNGNUqL/lewqe2v1q/oqLXDJKhfSsefH9C+x/J2+hNFKo2kmYMg==", + "version": "0.18.2", + "resolved": "https://registry.npmjs.org/@swirly/renderer-node/-/renderer-node-0.18.2.tgz", + "integrity": "sha512-Qcvqzjd6cEac/PZOv3mUAb2myEAnZbxCVb45kkjLRcs38Nqj4pq2aZH1E8E4OYR3XeZmHh8S7EtWO6k7phnqQg==", "dev": true, "requires": { - "@swirly/renderer": "^0.17.6", - "xmldom": "^0.5.0" + "@swirly/renderer": "^0.18.1", + "@xmldom/xmldom": "^0.8.2" } }, "@swirly/theme-default-base": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/theme-default-base/-/theme-default-base-0.17.6.tgz", - "integrity": "sha512-HbFMyL9xZjIzDhWzhAHoU7slfkET2euyLCB7lzOlRprPzfjatMTmCL30ji+Z6wRdQEITsrERCwiy5hfyo+rQEQ==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/theme-default-base/-/theme-default-base-0.18.1.tgz", + "integrity": "sha512-lX6ThJxHIsBj3So4BsVRanMf8Cw/1TupLUC9xeArGXzwLKCKAQXica8piDJd+spEU78A2r+g6AHgwqa4JuKZ6Q==", "dev": true }, "@swirly/theme-default-light": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/theme-default-light/-/theme-default-light-0.17.6.tgz", - "integrity": "sha512-KS6CmOcBqAVkJpx+K0rsNSSKGONrWsz7o7xT6Z4gOw0IFGwc/6/xjxvXGrMttbqutkGVv1vZEbl+wrAGDQ84WA==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/theme-default-light/-/theme-default-light-0.18.1.tgz", + "integrity": "sha512-PpjBvZykHOxLz1I+lTxfShvM3PUZBjD1bfnjArEP8mKu4Ng6ltQ/CZauQQDuvRvpC6CtnzBK1Gg/BNAjP93k0A==", "dev": true, "requires": { - "@swirly/theme-default-base": "^0.17.6" + "@swirly/theme-default-base": "^0.18.1" } }, "@swirly/types": { - "version": "0.17.6", - "resolved": "https://registry.npmjs.org/@swirly/types/-/types-0.17.6.tgz", - "integrity": "sha512-BHAEL95TTC1Trn2xHl5zJFyPDiDdGSo2GM32SLu1KC000vunjFPRvOVnfA8f9WflJz/It46ZiMaXl4a6g8F9ag==", + "version": "0.18.1", + "resolved": "https://registry.npmjs.org/@swirly/types/-/types-0.18.1.tgz", + "integrity": "sha512-TlXBe3nSfF5vuODhfP8ZKzmnqflWEZ7nXS6WQoS9loCOphMLBjycsm/01sM3SXrszCvbqRWFJ7nZrtSLQcgyMQ==", "dev": true }, "@szmarczak/http-timer": { @@ -29290,6 +29199,12 @@ "integrity": "sha512-5tXH6Bx/kNGd3MgffdmP4dy2Z+G4eaXw0SE81Tq3BNadtnMR5/ySMzX4SLEzHJzSmPNn4HIdpQsBvXMUykr58w==", "dev": true }, + "@types/lunr": { + "version": "2.3.4", + "resolved": "https://registry.npmjs.org/@types/lunr/-/lunr-2.3.4.tgz", + "integrity": "sha512-j4x4XJwZvorEUbA519VdQ5b9AOU9TSvfi8tvxMAfP8XzNLtFex7A8vFQwqOx3WACbV0KMXbACV3cZl4/gynQ7g==", + "dev": true + }, "@types/mdast": { "version": "3.0.10", "resolved": "https://registry.npmjs.org/@types/mdast/-/mdast-3.0.10.tgz", @@ -29524,6 +29439,12 @@ "resolved": "https://registry.npmjs.org/@webcomponents/custom-elements/-/custom-elements-1.5.0.tgz", "integrity": "sha512-c+7jPQCs9h/BYVcZ2Kna/3tsl3A/9EyXfvWjp5RiTDm1OpTcbZaCa1z4RNcTe/hUtXaqn64JjNW1yrWT+rZ8gg==" }, + "@xmldom/xmldom": { + "version": "0.8.2", + "resolved": "https://registry.npmjs.org/@xmldom/xmldom/-/xmldom-0.8.2.tgz", + "integrity": "sha512-+R0juSseERyoPvnBQ/cZih6bpF7IpCXlWbHRoCRzYzqpz6gWHOgf8o4MOEf6KBVuOyqU+gCNLkCWVIJAro8XyQ==", + "dev": true + }, "@xtuc/ieee754": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz", @@ -44492,6 +44413,12 @@ "integrity": "sha1-NbCYdbT/SfJqd35QmzCQoyJr8ks=", "dev": true }, + "stemmer": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/stemmer/-/stemmer-2.0.1.tgz", + "integrity": "sha512-bkWvSX2JR4nSZFfs113kd4C6X13bBBrg4fBKv2pVdzpdQI2LA5pZcWzTFNdkYsiUNl13E4EzymSRjZ0D55jBYg==", + "dev": true + }, "stream-shift": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", @@ -44853,70 +44780,6 @@ "upper-case": "^1.1.1" } }, - "swirly-parser": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-parser/-/swirly-parser-0.13.10.tgz", - "integrity": "sha512-EJvfYAHI10WDL60mpltu/lpNHtY77Elj2waAr8LPDwKreohk9B9XJ+m8V33i0iIKLsmDksIFl4l0RRVAQx7t7g==", - "dev": true, - "requires": { - "swirly-parser-rxjs": "^0.13.10" - } - }, - "swirly-parser-rxjs": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-parser-rxjs/-/swirly-parser-rxjs-0.13.10.tgz", - "integrity": "sha512-6wmdclC0zscSWjIKPgXkO5JurM/mVtV2tmzr1+j2ZEKiB4gKLV0QKSb4qGu87GtVYuzFjN4H4aCDH99Suq2GMA==", - "dev": true - }, - "swirly-renderer": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-renderer/-/swirly-renderer-0.13.10.tgz", - "integrity": "sha512-RSgdb9wE1SKcae49O8UjYaU8ayZWv6u/zRfkARGbuouYj2rXLceHyKAHBu9OAg4tRkyKZ1ioyxKFenHNeQE3rA==", - "dev": true, - "requires": { - "simple-sha1": "^3.0.1", - "swirly-theme-default-light": "^0.13.10" - } - }, - "swirly-renderer-node": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-renderer-node/-/swirly-renderer-node-0.13.10.tgz", - "integrity": "sha512-YL9umzpv8Nl9xAwxr1z+0CTqo94TcQFgRbzr/RJL1t6RG+b51q0x/O0oKG/jxIH9hnr6H3PJbzJwNrrwfHVsrQ==", - "dev": true, - "requires": { - "swirly-renderer": "^0.13.10", - "xmldom": "^0.3.0" - }, - "dependencies": { - "xmldom": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.3.0.tgz", - "integrity": "sha512-z9s6k3wxE+aZHgXYxSTpGDo7BYOUfJsIRyoZiX6HTjwpwfS2wpQBQKa2fD+ShLyPkqDYo5ud7KitmLZ2Cd6r0g==", - "dev": true - } - } - }, - "swirly-theme-default-base": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-theme-default-base/-/swirly-theme-default-base-0.13.10.tgz", - "integrity": "sha512-s1g9llE+fVdHemufzXcAM2p95OLCuwQ6/AB8tSAD9CStr0mZXWd6lPVGNw0i7T77bRe7TZirs/OeOq5gr2VgIw==", - "dev": true - }, - "swirly-theme-default-light": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-theme-default-light/-/swirly-theme-default-light-0.13.10.tgz", - "integrity": "sha512-2EXGLtcFKJHWN8snOQXqon0d3h2tS6Pu4v2TnLp7GdYU2ggEzqEKCfHWm5uEuW/j/HxBevlOBbdU0bZA1YRxJg==", - "dev": true, - "requires": { - "swirly-theme-default-base": "^0.13.10" - } - }, - "swirly-types": { - "version": "0.13.10", - "resolved": "https://registry.npmjs.org/swirly-types/-/swirly-types-0.13.10.tgz", - "integrity": "sha512-omsVUaHqCTcJV+kPt2iebg2XA4al2109MG6wkS/R2tBH1U1mv0TIaPeBSfbCctrt3EJKbazkJjY0axe1Fb1m1A==", - "dev": true - }, "symbol-observable": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/symbol-observable/-/symbol-observable-4.0.0.tgz", @@ -45490,12 +45353,6 @@ "integrity": "sha512-qLK/Xe9E2uzmYI3qLeOmI0tEOt+TBBQyUIAh4aAgU05FVYzeZrKUdkAZfBNVGRaHVgV0TDkdEngJSw/SyQchkQ==", "dev": true }, - "uglify-js": { - "version": "3.13.2", - "resolved": "https://registry.npmjs.org/uglify-js/-/uglify-js-3.13.2.tgz", - "integrity": "sha512-SbMu4D2Vo95LMC/MetNaso1194M1htEA+JrqE9Hk+G2DhI+itfS9TRu9ZKeCahLDNa/J3n4MqUJ/fOHMzQpRWw==", - "dev": true - }, "ultron": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ultron/-/ultron-1.1.1.tgz", @@ -46804,12 +46661,6 @@ "integrity": "sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw==", "dev": true }, - "xmldom": { - "version": "0.5.0", - "resolved": "https://registry.npmjs.org/xmldom/-/xmldom-0.5.0.tgz", - "integrity": "sha512-Foaj5FXVzgn7xFzsKeNIde9g6aFBxTPi37iwsno8QvApmtg7KYrr+OPyRHcJF7dud2a5nGRBXK3n0dL62Gf7PA==", - "dev": true - }, "xregexp": { "version": "4.4.1", "resolved": "https://registry.npmjs.org/xregexp/-/xregexp-4.4.1.tgz", diff --git a/docs_app/package.json b/docs_app/package.json index 03626badd1..8f375a1629 100644 --- a/docs_app/package.json +++ b/docs_app/package.json @@ -6,7 +6,6 @@ "author": "RxJS", "license": "MIT", "scripts": { - "postinstall": "uglifyjs node_modules/lunr/lunr.js -c -m -o src/assets/js/lunr.min.js --source-map", "ng": "ng", "firebase": "firebase", "start": "ng serve --configuration=fast", @@ -73,6 +72,7 @@ "@swirly/types": "^0.18.1", "@types/jasmine": "~3.6.0", "@types/jasminewd2": "^2.0.3", + "@types/lunr": "^2.3.3", "@types/node": "^12.11.1", "@types/svgo": "^1.3.3", "archiver": "^3.0.0", @@ -118,13 +118,13 @@ "rimraf": "^2.6.1", "semver": "^6.1.1", "shelljs": "^0.8.3", + "stemmer": "^2.0.0", "svgo": "^1.3.2", "svgson": "^4.1.0", "tree-kill": "^1.2.2", "ts-node": "^8.2.0", "tslint": "~6.1.0", "typescript": "4.5.4", - "uglify-js": "^3.6.0", "unist-util-filter": "^1.0.2", "unist-util-source": "^1.0.5", "unist-util-visit": "^1.4.1", diff --git a/docs_app/src/app/app.component.ts b/docs_app/src/app/app.component.ts index 04e5abd80c..a0a702360e 100644 --- a/docs_app/src/app/app.component.ts +++ b/docs_app/src/app/app.component.ts @@ -200,7 +200,7 @@ export class AppComponent implements OnInit { // Do not initialize the search on browsers that lack web worker support if ('Worker' in window) { // Delay initialization by up to 2 seconds - this.searchService.initWorker('app/search/search-worker.js', 2000); + this.searchService.initWorker(2000); } this.onResize(window.innerWidth); diff --git a/docs_app/src/app/search/search-worker.js b/docs_app/src/app/search/search-worker.js deleted file mode 100644 index d29744c608..0000000000 --- a/docs_app/src/app/search/search-worker.js +++ /dev/null @@ -1,103 +0,0 @@ -'use strict'; - -/* eslint-env worker */ -/* global importScripts, lunr */ - -var SEARCH_TERMS_URL = '/generated/docs/app/search-data.json'; - -// NOTE: This needs to be kept in sync with `ngsw-config.json`. -importScripts('/assets/js/lunr.min.js'); - -var index; -var pages /* : SearchInfo */ = {}; - -// interface SearchInfo { -// [key: string]: PageInfo; -// } - -// interface PageInfo { -// path: string; -// type: string, -// titleWords: string; -// keyWords: string; -// } - -self.onmessage = handleMessage; - -// Create the lunr index - the docs should be an array of objects, each object containing -// the path and search terms for a page -function createIndex(addFn) { - return lunr(/** @this */function() { - this.pipeline.remove(lunr.stopWordFilter); - this.ref('path'); - this.field('titleWords', {boost: 100}); - this.field('headingWords', {boost: 50}); - this.field('members', {boost: 40}); - this.field('keywords', {boost: 20}); - addFn(this); - }); -} - -// The worker receives a message to load the index and to query the index -function handleMessage(message) { - var type = message.data.type; - var id = message.data.id; - var payload = message.data.payload; - switch(type) { - case 'load-index': - makeRequest(SEARCH_TERMS_URL, function(searchInfo) { - index = createIndex(loadIndex(searchInfo)); - self.postMessage({type: type, id: id, payload: true}); - }); - break; - case 'query-index': - self.postMessage({type: type, id: id, payload: {query: payload, results: queryIndex(payload)}}); - break; - default: - self.postMessage({type: type, id: id, payload: {error: 'invalid message type'}}) - } -} - -// Use XHR to make a request to the server -function makeRequest(url, callback) { - - // The JSON file that is loaded should be an array of PageInfo: - var searchDataRequest = new XMLHttpRequest(); - searchDataRequest.onload = function() { - callback(JSON.parse(this.responseText)); - }; - searchDataRequest.open('GET', url); - searchDataRequest.send(); -} - - -// Create the search index from the searchInfo which contains the information about each page to be indexed -function loadIndex(searchInfo /*: SearchInfo */) { - return function(index) { - // Store the pages data to be used in mapping query results back to pages - // Add search terms from each page to the search index - searchInfo.forEach(function(page /*: PageInfo */) { - index.add(page); - pages[page.path] = page; - }); - }; -} - -// Query the index and return the processed results -function queryIndex(query) { - try { - if (query.length) { - // Add a relaxed search in the title for the first word in the query - // E.g. if the search is "ngCont guide" then we search for "ngCont guide titleWords:ngCont*" - var titleQuery = 'titleWords:*' + query.split(' ', 1)[0] + '*'; - var results = index.search(query + ' ' + titleQuery); - // Map the hits into info about each page to be returned as results - return results.map(function(hit) { return pages[hit.ref]; }); - } - } catch(e) { - // If the search query cannot be parsed the index throws an error - // Log it and recover - console.log(e); - } - return []; -} diff --git a/docs_app/src/app/search/search.service.ts b/docs_app/src/app/search/search.service.ts index cd37900402..3455d60bed 100644 --- a/docs_app/src/app/search/search.service.ts +++ b/docs_app/src/app/search/search.service.ts @@ -22,23 +22,19 @@ export class SearchService { * initial rendering of the web page. Triggering a search will override this delay and cause the index to be * loaded immediately. * - * @param workerUrl the url of the WebWorker script that runs the searches * @param initDelay the number of milliseconds to wait before we load the WebWorker and generate the search index */ - initWorker(workerUrl: string, initDelay: number) { + initWorker(initDelay: number) { // Wait for the initDelay or the first search - const ready = this.ready = race( - timer(initDelay), - this.searchesSubject.asObservable().pipe(first()), - ) - .pipe( - concatMap(() => { - // Create the worker and load the index - this.worker = WebWorkerClient.create(workerUrl, this.zone); - return this.worker.sendMessage('load-index'); - }), - publishReplay(1), - ); + const ready = (this.ready = race(timer(initDelay), this.searchesSubject.asObservable().pipe(first())).pipe( + concatMap(() => { + // Create the worker and load the index + const worker = new Worker(new URL('./search.worker', import.meta.url), { type: 'module' }); + this.worker = WebWorkerClient.create(worker, this.zone); + return this.worker.sendMessage('load-index'); + }), + publishReplay(1) + )); // Connect to the observable to kick off the timer (ready as ConnectableObservable).connect(); @@ -47,6 +43,7 @@ export class SearchService { /** * Search the index using the given query and emit results on the observable that is returned. + * * @param query The query to run against the index. * @returns an observable collection of search results */ diff --git a/docs_app/src/app/search/search.worker.ts b/docs_app/src/app/search/search.worker.ts new file mode 100644 index 0000000000..e3b4d9935c --- /dev/null +++ b/docs_app/src/app/search/search.worker.ts @@ -0,0 +1,174 @@ +/* eslint-env worker */ +/// +import * as lunr from 'lunr'; + +import { WebWorkerMessage } from '../shared/web-worker-message'; + +const SEARCH_TERMS_URL = '/generated/docs/app/search-data.json'; +let index: lunr.Index; +const pageMap: SearchInfo = {}; + +interface SearchInfo { + [key: string]: PageInfo; +} + +interface PageInfo { + path: string; + type: string; + title: string; + headings: string; + keywords: string; + members: string; + topics: string; +} + +interface EncodedPages { + dictionary: string; + pages: EncodedPage[]; +} + +interface EncodedPage { + path: string; + type: string; + title: string; + headings: number[]; + keywords: number[]; + members: number[]; + topics: string; +} + +addEventListener('message', handleMessage); + +const customLunr = function (config: lunr.ConfigFunction) { + var builder = new lunr.Builder(); + builder.pipeline.add(lunr.trimmer, lunr.stemmer); + builder.searchPipeline.add(lunr.stemmer); + config.call(builder, builder); + return builder.build(); +}; + +// Create the lunr index - the docs should be an array of objects, each object containing +// the path and search terms for a page +function createIndex(loadIndexFn: IndexLoader): lunr.Index { + // The lunr typings are missing QueryLexer so we have to add them here manually. + const queryLexer = (lunr as any as { QueryLexer: { termSeparator: RegExp } }).QueryLexer; + queryLexer.termSeparator = lunr.tokenizer.separator = /\s+/; + return customLunr(function () { + this.ref('path'); + this.field('topics', { boost: 15 }); + this.field('title', { boost: 10 }); + this.field('headings', { boost: 5 }); + this.field('members', { boost: 4 }); + this.field('keywords', { boost: 2 }); + loadIndexFn(this); + }); +} + +// The worker receives a message to load the index and to query the index +function handleMessage(message: { data: WebWorkerMessage }): void { + const type = message.data.type; + const id = message.data.id; + const payload = message.data.payload; + switch (type) { + case 'load-index': + makeRequest(SEARCH_TERMS_URL, (encodedPages: EncodedPages) => { + index = createIndex(loadIndex(encodedPages)); + postMessage({ type, id, payload: true }); + }); + break; + case 'query-index': + postMessage({ type, id, payload: { query: payload, results: queryIndex(payload) } }); + break; + default: + postMessage({ type, id, payload: { error: 'invalid message type' } }); + } +} + +// Use XHR to make a request to the server +function makeRequest(url: string, callback: (response: any) => void): void { + // The JSON file that is loaded should be an array of PageInfo: + const searchDataRequest = new XMLHttpRequest(); + searchDataRequest.onload = function () { + callback(JSON.parse(this.responseText)); + }; + searchDataRequest.open('GET', url); + searchDataRequest.send(); +} + +// Create the search index from the searchInfo which contains the information about each page to be +// indexed +function loadIndex({ dictionary, pages }: EncodedPages): IndexLoader { + const dictionaryArray = dictionary.split(' '); + return (indexBuilder: lunr.Builder) => { + // Store the pages data to be used in mapping query results back to pages + // Add search terms from each page to the search index + pages.forEach((encodedPage) => { + const page = decodePage(encodedPage, dictionaryArray); + indexBuilder.add(page); + pageMap[page.path] = page; + }); + }; +} + +function decodePage(encodedPage: EncodedPage, dictionary: string[]): PageInfo { + return { + ...encodedPage, + headings: encodedPage.headings?.map((i) => dictionary[i]).join(' ') ?? '', + keywords: encodedPage.keywords?.map((i) => dictionary[i]).join(' ') ?? '', + members: encodedPage.members?.map((i) => dictionary[i]).join(' ') ?? '', + }; +} + +// Query the index and return the processed results +function queryIndex(query: string): PageInfo[] { + // Strip off quotes + query = query.replace(/^["']|['"]$/g, ''); + try { + if (query.length) { + let results = index.query((queryBuilder) => { + queryBuilder.term(lunr.tokenizer(query), { + fields: ['title'], + wildcard: lunr.Query.wildcard.TRAILING | lunr.Query.wildcard.LEADING, + usePipeline: true, + presence: lunr.Query.presence.REQUIRED, + }); + }); + + if (results.length === 0) { + // First try a query where every term must be present + // (see https://lunrjs.com/guides/searching.html#term-presence) + results = index.query((queryBuilder) => { + const tokens = lunr.tokenizer(query); + for (const token of tokens) { + queryBuilder.term(token, { + usePipeline: true, + presence: lunr.Query.presence.REQUIRED, + }); + } + }); + } + + // If that was too restrictive just query for any term to be present + if (results.length === 0) { + results = index.search(query); + } + + // If that is still too restrictive then search in the title for the first word in the query + if (results.length === 0) { + // E.g. if the search is "ngCont guide" then we search for "ngCont guide title:*ngCont*" + const titleQuery = 'title:*' + query.split(' ', 1)[0] + '*'; + results = index.search(query + ' ' + titleQuery); + } + + // Map the hits into info about each page to be returned as results + return results.map((hit) => pageMap[hit.ref]); + } + } catch (e) { + // If the search query cannot be parsed the index throws an error + // Log it and recover + console.error(e); + } + return []; +} + +type IndexLoader = (indexBuilder: lunr.Builder) => void; diff --git a/docs_app/src/app/shared/web-worker-message.ts b/docs_app/src/app/shared/web-worker-message.ts new file mode 100644 index 0000000000..36d2a69e36 --- /dev/null +++ b/docs_app/src/app/shared/web-worker-message.ts @@ -0,0 +1,5 @@ +export interface WebWorkerMessage { + type: string; + payload: any; + id?: number; +} diff --git a/docs_app/src/app/shared/web-worker.ts b/docs_app/src/app/shared/web-worker.ts index 464e32fe07..933a73d723 100644 --- a/docs_app/src/app/shared/web-worker.ts +++ b/docs_app/src/app/shared/web-worker.ts @@ -4,33 +4,25 @@ Use of this source code is governed by an MIT-style license that can be found in the LICENSE file at http://angular.io/license */ -import {NgZone} from '@angular/core'; -import {Observable} from 'rxjs'; - -export interface WebWorkerMessage { - type: string; - payload: any; - id?: number; -} +import { NgZone } from '@angular/core'; +import { Observable } from 'rxjs'; +import { WebWorkerMessage } from './web-worker-message'; export class WebWorkerClient { private nextId = 0; - static create(workerUrl: string, zone: NgZone) { - return new WebWorkerClient(new Worker(workerUrl), zone); + static create(worker: Worker, zone: NgZone) { + return new WebWorkerClient(worker, zone); } - private constructor(private worker: Worker, private zone: NgZone) { - } + private constructor(private worker: Worker, private zone: NgZone) {} sendMessage(type: string, payload?: any): Observable { - - return new Observable(subscriber => { - + return new Observable((subscriber) => { const id = this.nextId++; const handleMessage = (response: MessageEvent) => { - const {type: responseType, id: responseId, payload: responsePayload} = response.data as WebWorkerMessage; + const { type: responseType, id: responseId, payload: responsePayload } = response.data as WebWorkerMessage; if (type === responseType && id === responseId) { this.zone.run(() => { subscriber.next(responsePayload); @@ -49,7 +41,7 @@ export class WebWorkerClient { this.worker.addEventListener('error', handleError); // Post the message to the web worker - this.worker.postMessage({type, id, payload}); + this.worker.postMessage({ type, id, payload }); // At completion/error unwire the event listeners return () => { diff --git a/docs_app/tools/transforms/angular-base-package/ignore-words.json b/docs_app/tools/transforms/angular-base-package/ignore-words.json new file mode 100644 index 0000000000..3d71c9915d --- /dev/null +++ b/docs_app/tools/transforms/angular-base-package/ignore-words.json @@ -0,0 +1,703 @@ +[ + "a", + "able", + "about", + "above", + "abst", + "accordance", + "according", + "accordingly", + "across", + "act", + "actually", + "added", + "adj", + "adopted", + "affected", + "affecting", + "affects", + "after", + "afterwards", + "again", + "against", + "ah", + "all", + "almost", + "alone", + "along", + "already", + "also", + "although", + "always", + "am", + "among", + "amongst", + "an", + "and", + "announce", + "another", + "any", + "anybody", + "anyhow", + "anymore", + "anyone", + "anything", + "anyway", + "anyways", + "anywhere", + "apparently", + "approximately", + "are", + "aren", + "arent", + "arise", + "around", + "as", + "aside", + "ask", + "asking", + "at", + "auth", + "available", + "away", + "awfully", + "b", + "back", + "be", + "became", + "because", + "become", + "becomes", + "becoming", + "been", + "before", + "beforehand", + "begin", + "beginning", + "beginnings", + "begins", + "behind", + "being", + "believe", + "below", + "beside", + "besides", + "between", + "beyond", + "biol", + "both", + "brief", + "briefly", + "but", + "by", + "c", + "ca", + "came", + "can", + "cannot", + "can't", + "cant", + "cause", + "causes", + "certain", + "certainly", + "co", + "com", + "come", + "comes", + "contain", + "containing", + "contains", + "could", + "couldnt", + "d", + "date", + "did", + "didn't", + "didnt", + "different", + "do", + "does", + "doesn't", + "doesnt", + "doing", + "done", + "don't", + "dont", + "down", + "downwards", + "due", + "during", + "e", + "each", + "ed", + "edu", + "effect", + "eg", + "eight", + "eighty", + "either", + "else", + "elsewhere", + "end", + "ending", + "enough", + "especially", + "et", + "et-al", + "etc", + "even", + "ever", + "every", + "everybody", + "everyone", + "everything", + "everywhere", + "ex", + "except", + "f", + "far", + "few", + "ff", + "fifth", + "first", + "five", + "fix", + "followed", + "following", + "follows", + "for", + "former", + "formerly", + "forth", + "found", + "four", + "from", + "further", + "furthermore", + "g", + "gave", + "get", + "gets", + "getting", + "give", + "given", + "gives", + "giving", + "go", + "goes", + "gone", + "got", + "gotten", + "h", + "had", + "happens", + "hardly", + "has", + "hasn't", + "hasnt", + "have", + "haven't", + "havent", + "having", + "he", + "hed", + "hence", + "her", + "here", + "hereafter", + "hereby", + "herein", + "heres", + "hereupon", + "hers", + "herself", + "hes", + "hi", + "hid", + "him", + "himself", + "his", + "hither", + "home", + "how", + "howbeit", + "however", + "hundred", + "i", + "id", + "ie", + "if", + "i'll", + "ill", + "im", + "immediate", + "immediately", + "importance", + "important", + "in", + "inc", + "indeed", + "index", + "information", + "instead", + "into", + "invention", + "inward", + "is", + "isn't", + "isnt", + "it", + "itd", + "it'll", + "itll", + "its", + "itself", + "i've", + "ive", + "j", + "just", + "k", + "keep", + "keeps", + "kept", + "keys", + "kg", + "km", + "know", + "known", + "knows", + "l", + "largely", + "last", + "lately", + "later", + "latter", + "latterly", + "least", + "less", + "lest", + "let", + "lets", + "like", + "liked", + "likely", + "line", + "little", + "'ll", + "'ll", + "look", + "looking", + "looks", + "ltd", + "m", + "made", + "mainly", + "make", + "makes", + "many", + "may", + "maybe", + "me", + "mean", + "means", + "meantime", + "meanwhile", + "merely", + "mg", + "might", + "million", + "miss", + "ml", + "more", + "moreover", + "most", + "mostly", + "mr", + "mrs", + "much", + "mug", + "must", + "my", + "myself", + "n", + "na", + "name", + "namely", + "nay", + "nd", + "near", + "nearly", + "necessarily", + "necessary", + "need", + "needs", + "neither", + "never", + "nevertheless", + "new", + "next", + "nine", + "ninety", + "no", + "nobody", + "non", + "none", + "nonetheless", + "noone", + "nor", + "normally", + "nos", + "not", + "noted", + "nothing", + "now", + "nowhere", + "o", + "obtain", + "obtained", + "obviously", + "of", + "off", + "often", + "oh", + "ok", + "okay", + "old", + "omitted", + "on", + "once", + "one", + "ones", + "only", + "onto", + "or", + "ord", + "other", + "others", + "otherwise", + "ought", + "our", + "ours", + "ourselves", + "out", + "outside", + "over", + "overall", + "owing", + "own", + "p", + "page", + "pages", + "part", + "particular", + "particularly", + "past", + "per", + "perhaps", + "placed", + "please", + "plus", + "poorly", + "possible", + "possibly", + "potentially", + "pp", + "predominantly", + "present", + "previously", + "primarily", + "probably", + "promptly", + "proud", + "provides", + "put", + "q", + "que", + "quickly", + "quite", + "qv", + "r", + "ran", + "rather", + "rd", + "re", + "readily", + "really", + "recent", + "recently", + "ref", + "refs", + "regarding", + "regardless", + "regards", + "related", + "relatively", + "research", + "respectively", + "resulted", + "resulting", + "results", + "right", + "run", + "s", + "said", + "same", + "saw", + "say", + "saying", + "says", + "sec", + "section", + "see", + "seeing", + "seem", + "seemed", + "seeming", + "seems", + "seen", + "self", + "selves", + "sent", + "seven", + "several", + "shall", + "she", + "shed", + "she'll", + "shell", + "shes", + "should", + "shouldn't", + "shouldnt", + "show", + "showed", + "shown", + "showns", + "shows", + "significant", + "significantly", + "similar", + "similarly", + "since", + "six", + "slightly", + "so", + "some", + "somebody", + "somehow", + "someone", + "somethan", + "something", + "sometime", + "sometimes", + "somewhat", + "somewhere", + "soon", + "sorry", + "specifically", + "specified", + "specify", + "specifying", + "state", + "states", + "still", + "stop", + "strongly", + "sub", + "substantially", + "successfully", + "such", + "sufficiently", + "suggest", + "sup", + "sure", + "t", + "take", + "taken", + "taking", + "tell", + "tends", + "th", + "than", + "thank", + "thanks", + "thanx", + "that", + "that'll", + "thatll", + "thats", + "that've", + "thatve", + "the", + "their", + "theirs", + "them", + "themselves", + "then", + "thence", + "there", + "thereafter", + "thereby", + "thered", + "therefore", + "therein", + "there'll", + "therell", + "thereof", + "therere", + "theres", + "thereto", + "thereupon", + "there've", + "thereve", + "these", + "they", + "theyd", + "they'll", + "theyll", + "theyre", + "they've", + "theyve", + "think", + "this", + "those", + "thou", + "though", + "thoughh", + "thousand", + "throug", + "through", + "throughout", + "thru", + "thus", + "til", + "tip", + "to", + "together", + "too", + "took", + "toward", + "towards", + "tried", + "tries", + "truly", + "try", + "trying", + "ts", + "twice", + "two", + "u", + "un", + "under", + "unfortunately", + "unless", + "unlike", + "unlikely", + "until", + "unto", + "up", + "upon", + "ups", + "us", + "use", + "used", + "useful", + "usefully", + "usefulness", + "uses", + "using", + "usually", + "v", + "value", + "various", + "'ve", + "'ve", + "very", + "via", + "viz", + "vol", + "vols", + "vs", + "w", + "want", + "wants", + "was", + "wasn't", + "wasnt", + "way", + "we", + "wed", + "welcome", + "we'll", + "well", + "went", + "were", + "weren't", + "werent", + "we've", + "weve", + "what", + "whatever", + "what'll", + "whatll", + "whats", + "when", + "whence", + "whenever", + "where", + "whereafter", + "whereas", + "whereby", + "wherein", + "wheres", + "whereupon", + "wherever", + "whether", + "which", + "while", + "whim", + "whither", + "who", + "whod", + "whoever", + "whole", + "who'll", + "wholl", + "whom", + "whomever", + "whos", + "whose", + "why", + "widely", + "will", + "willing", + "wish", + "with", + "within", + "without", + "won't", + "wont", + "words", + "would", + "wouldn't", + "wouldnt", + "www", + "x", + "y", + "yes", + "yet", + "you", + "youd", + "you'll", + "youll", + "your", + "youre", + "yours", + "yourself", + "yourselves", + "you've", + "youve", + "z", + "zero" +] diff --git a/docs_app/tools/transforms/angular-base-package/ignore.words b/docs_app/tools/transforms/angular-base-package/ignore.words deleted file mode 100644 index 82b9f2fc3f..0000000000 --- a/docs_app/tools/transforms/angular-base-package/ignore.words +++ /dev/null @@ -1,701 +0,0 @@ -a -able -about -above -abst -accordance -according -accordingly -across -act -actually -added -adj -adopted -affected -affecting -affects -after -afterwards -again -against -ah -all -almost -alone -along -already -also -although -always -am -among -amongst -an -and -announce -another -any -anybody -anyhow -anymore -anyone -anything -anyway -anyways -anywhere -apparently -approximately -are -aren -arent -arise -around -as -aside -ask -asking -at -auth -available -away -awfully -b -back -be -became -because -become -becomes -becoming -been -before -beforehand -begin -beginning -beginnings -begins -behind -being -believe -below -beside -besides -between -beyond -biol -both -brief -briefly -but -by -c -ca -came -can -cannot -can't -cant -cause -causes -certain -certainly -co -com -come -comes -contain -containing -contains -could -couldnt -d -date -did -didn't -didnt -different -do -does -doesn't -doesnt -doing -done -don't -dont -down -downwards -due -during -e -each -ed -edu -effect -eg -eight -eighty -either -else -elsewhere -end -ending -enough -especially -et -et-al -etc -even -ever -every -everybody -everyone -everything -everywhere -ex -except -f -far -few -ff -fifth -first -five -fix -followed -following -follows -for -former -formerly -forth -found -four -from -further -furthermore -g -gave -get -gets -getting -give -given -gives -giving -go -goes -gone -got -gotten -h -had -happens -hardly -has -hasn't -hasnt -have -haven't -havent -having -he -hed -hence -her -here -hereafter -hereby -herein -heres -hereupon -hers -herself -hes -hi -hid -him -himself -his -hither -home -how -howbeit -however -hundred -i -id -ie -if -i'll -ill -im -immediate -immediately -importance -important -in -inc -indeed -index -information -instead -into -invention -inward -is -isn't -isnt -it -itd -it'll -itll -its -itself -i've -ive -j -just -k -keep -keeps -kept -keys -kg -km -know -known -knows -l -largely -last -lately -later -latter -latterly -least -less -lest -let -lets -like -liked -likely -line -little -'ll -'ll -look -looking -looks -ltd -m -made -mainly -make -makes -many -may -maybe -me -mean -means -meantime -meanwhile -merely -mg -might -million -miss -ml -more -moreover -most -mostly -mr -mrs -much -mug -must -my -myself -n -na -name -namely -nay -nd -near -nearly -necessarily -necessary -need -needs -neither -never -nevertheless -new -next -nine -ninety -no -nobody -non -none -nonetheless -noone -nor -normally -nos -not -noted -nothing -now -nowhere -o -obtain -obtained -obviously -of -off -often -oh -ok -okay -old -omitted -on -once -one -ones -only -onto -or -ord -other -others -otherwise -ought -our -ours -ourselves -out -outside -over -overall -owing -own -p -page -pages -part -particular -particularly -past -per -perhaps -placed -please -plus -poorly -possible -possibly -potentially -pp -predominantly -present -previously -primarily -probably -promptly -proud -provides -put -q -que -quickly -quite -qv -r -ran -rather -rd -re -readily -really -recent -recently -ref -refs -regarding -regardless -regards -related -relatively -research -respectively -resulted -resulting -results -right -run -s -said -same -saw -say -saying -says -sec -section -see -seeing -seem -seemed -seeming -seems -seen -self -selves -sent -seven -several -shall -she -shed -she'll -shell -shes -should -shouldn't -shouldnt -show -showed -shown -showns -shows -significant -significantly -similar -similarly -since -six -slightly -so -some -somebody -somehow -someone -somethan -something -sometime -sometimes -somewhat -somewhere -soon -sorry -specifically -specified -specify -specifying -state -states -still -stop -strongly -sub -substantially -successfully -such -sufficiently -suggest -sup -sure -t -take -taken -taking -tell -tends -th -than -thank -thanks -thanx -that -that'll -thatll -thats -that've -thatve -the -their -theirs -them -themselves -then -thence -there -thereafter -thereby -thered -therefore -therein -there'll -therell -thereof -therere -theres -thereto -thereupon -there've -thereve -these -they -theyd -they'll -theyll -theyre -they've -theyve -think -this -those -thou -though -thoughh -thousand -throug -through -throughout -thru -thus -til -tip -to -together -too -took -toward -towards -tried -tries -truly -try -trying -ts -twice -two -u -un -under -unfortunately -unless -unlike -unlikely -until -unto -up -upon -ups -us -use -used -useful -usefully -usefulness -uses -using -usually -v -value -various -'ve -'ve -very -via -viz -vol -vols -vs -w -want -wants -was -wasn't -wasnt -way -we -wed -welcome -we'll -well -went -were -weren't -werent -we've -weve -what -whatever -what'll -whatll -whats -when -whence -whenever -where -whereafter -whereas -whereby -wherein -wheres -whereupon -wherever -whether -which -while -whim -whither -who -whod -whoever -whole -who'll -wholl -whom -whomever -whos -whose -why -widely -will -willing -wish -with -within -without -won't -wont -words -would -wouldn't -wouldnt -www -x -y -yes -yet -you -youd -you'll -youll -your -youre -yours -yourself -yourselves -you've -youve -z -zero diff --git a/docs_app/tools/transforms/angular-base-package/index.js b/docs_app/tools/transforms/angular-base-package/index.js index 71b616dbe0..1c01daadb4 100644 --- a/docs_app/tools/transforms/angular-base-package/index.js +++ b/docs_app/tools/transforms/angular-base-package/index.js @@ -64,8 +64,17 @@ module.exports = new Package('angular-base', [ readFilesProcessor.basePath = PROJECT_ROOT; readFilesProcessor.sourceFiles = []; - generateKeywordsProcessor.ignoreWordsFile = path.resolve(__dirname, 'ignore.words'); - generateKeywordsProcessor.propertiesToIgnore = ['renderedContent']; + generateKeywordsProcessor.ignoreWords = require(path.resolve(__dirname, 'ignore-words')); + generateKeywordsProcessor.docTypesToIgnore = [ + undefined, + 'json-doc', + 'api-list-data', + 'api-list-data', + 'contributors-json', + 'navigation-json', + 'announcements-json', + ]; + generateKeywordsProcessor.propertiesToIgnore = ['basePath', 'renderedContent', 'docType', 'searchTitle']; }) // Where do we write the output files? @@ -126,7 +135,15 @@ module.exports = new Package('angular-base', [ generateKeywordsProcessor.outputFolder = 'app'; }) - .config(function (postProcessHtml, addImageDimensions, autoLinkCode, filterPipes, filterAmbiguousDirectiveAliases, filterFromInImports, filterNeverAsGeneric) { + .config(function ( + postProcessHtml, + addImageDimensions, + autoLinkCode, + filterPipes, + filterAmbiguousDirectiveAliases, + filterFromInImports, + filterNeverAsGeneric + ) { addImageDimensions.basePath = path.resolve(AIO_PATH, 'src'); autoLinkCode.customFilters = [filterPipes, filterAmbiguousDirectiveAliases]; autoLinkCode.wordFilters = [filterFromInImports, filterNeverAsGeneric]; diff --git a/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.js b/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.js index 7247b93a57..db11819aed 100644 --- a/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.js +++ b/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.js @@ -1,7 +1,4 @@ -'use strict'; - -var fs = require('fs'); -var path = require('canonical-path'); +/* eslint-disable */ /** * @dgProcessor generateKeywordsProcessor @@ -10,129 +7,100 @@ var path = require('canonical-path'); * a new document that will be rendered as a JavaScript file containing all * this data. */ -module.exports = function generateKeywordsProcessor(log, readFilesProcessor) { +module.exports = function generateKeywordsProcessor(log) { return { - ignoreWordsFile: undefined, + ignoreWords: [], propertiesToIgnore: [], docTypesToIgnore: [], outputFolder: '', $validate: { - ignoreWordsFile: {}, + ignoreWords: {}, docTypesToIgnore: {}, propertiesToIgnore: {}, - outputFolder: {presence: true} + outputFolder: { presence: true }, }, $runAfter: ['postProcessHtml'], $runBefore: ['writing-files'], - $process: function(docs) { - - // Keywords to ignore - var wordsToIgnore = []; - var propertiesToIgnore; - var docTypesToIgnore; + async $process(docs) { + const { stemmer: stem } = await import('stemmer'); - // Keywords start with "ng:" or one of $, _ or a letter - var KEYWORD_REGEX = /^((ng:|[$_a-z])[\w\-_]+)/; + const dictionary = new Map(); - // Load up the keywords to ignore, if specified in the config - if (this.ignoreWordsFile) { - var ignoreWordsPath = path.resolve(readFilesProcessor.basePath, this.ignoreWordsFile); - wordsToIgnore = fs.readFileSync(ignoreWordsPath, 'utf8').toString().split(/[,\s\n\r]+/gm); + const emptySet = new Set(); - log.debug('Loaded ignore words from "' + ignoreWordsPath + '"'); - log.silly(wordsToIgnore); - } - - propertiesToIgnore = convertToMap(this.propertiesToIgnore); + // Keywords to ignore + const ignoreWords = new Set(this.ignoreWords); + log.debug('Words to ignore', ignoreWords); + const propertiesToIgnore = new Set(this.propertiesToIgnore); log.debug('Properties to ignore', propertiesToIgnore); - docTypesToIgnore = convertToMap(this.docTypesToIgnore); + const docTypesToIgnore = new Set(this.docTypesToIgnore); log.debug('Doc types to ignore', docTypesToIgnore); - var ignoreWordsMap = convertToMap(wordsToIgnore); - - // If the heading contains a name starting with ng, e.g. "ngController", then add the - // name without the ng to the text, e.g. "controller". - function preprocessText(text) { - return text.replace(/(^|\s)([nN]g([A-Z]\w*))/g, '$1$2 $3'); - } - - function extractWords(text, words, keywordMap) { - var tokens = preprocessText(text).toLowerCase().split(/[.\s,`'"#]+/mg); - tokens.forEach(function(token) { - var match = token.match(KEYWORD_REGEX); - if (match) { - var key = match[1]; - if (!keywordMap[key]) { - keywordMap[key] = true; - words.push(key); - } - } - }); - } - - const filteredDocs = docs - // We are not interested in some docTypes - .filter(function(doc) { return !docTypesToIgnore[doc.docType]; }) - // Ignore internals and private exports (indicated by the ɵ prefix) - .filter(function(doc) { return !doc.internal && !doc.privateExport; }); - - - filteredDocs.forEach(function(doc) { - - var words = []; - var keywordMap = Object.assign({}, ignoreWordsMap); - var members = []; - var membersMap = Object.assign({}, ignoreWordsMap); - const headingWords = []; - const headingWordMap = Object.assign({}, ignoreWordsMap); - + // We are not interested in some docTypes + .filter((doc) => !docTypesToIgnore.has(doc.docType)) + // Ignore internals and private exports (indicated by the ɵ prefix) + .filter((doc) => !doc.internal && !doc.privateExport) + // Ignore duplicates and remove the `/api/operators/` path entries from the search results + .filter((doc) => doc.path.indexOf('api/operators/') !== 0); + + for (const doc of filteredDocs) { // Search each top level property of the document for search terms - Object.keys(doc).forEach(function(key) { + let mainTokens = []; + for (const key of Object.keys(doc)) { const value = doc[key]; - - if (isString(value) && !propertiesToIgnore[key]) { - extractWords(value, words, keywordMap); + if (isString(value) && !propertiesToIgnore.has(key)) { + mainTokens.push(...tokenize(value, ignoreWords, dictionary)); } + } - // Special case properties that contain content relating to "members" - // of a doc that represents, say, a class or interface - if (key === 'members' || key === 'statics') { - value.forEach(function(member) { extractWords(member.name, members, membersMap); }); - } - }); + const memberTokens = extractMemberTokens(doc, dictionary); // Extract all the keywords from the headings + let headingTokens = []; if (doc.vFile && doc.vFile.headings) { - Object.keys(doc.vFile.headings).forEach(function(headingTag) { - doc.vFile.headings[headingTag].forEach(function(headingText) { - extractWords(headingText, headingWords, headingWordMap); - }); - }); + for (const headingTag of Object.keys(doc.vFile.headings)) { + for (const headingText of doc.vFile.headings[headingTag]) { + headingTokens.push(...tokenize(headingText, ignoreWords, dictionary)); + } + } } // Extract the title to use in searches - doc.searchTitle = doc.searchTitle || doc.title || doc.vFile && doc.vFile.title || doc.name || ''; + doc.searchTitle = doc.searchTitle || doc.title || (doc.vFile && doc.vFile.title) || doc.name || ''; // Attach all this search data to the document - doc.searchTerms = { - titleWords: preprocessText(doc.searchTitle), - headingWords: headingWords.sort().join(' '), - keywords: words.sort().join(' '), - members: members.sort().join(' ') - }; - - }); + doc.searchTerms = {}; + if (headingTokens.length > 0) { + doc.searchTerms.headings = headingTokens; + } + if (mainTokens.length > 0) { + doc.searchTerms.keywords = mainTokens; + } + if (memberTokens.length > 0) { + doc.searchTerms.members = memberTokens; + } + if (doc.searchKeywords) { + doc.searchTerms.topics = doc.searchKeywords.trim(); + } + } // Now process all the search data and collect it up to be used in creating a new document - var searchData = filteredDocs.map(function(page) { - // Copy the properties from the searchTerms object onto the search data object - return Object.assign({ - path: page.path, - title: page.searchTitle, - type: page.docType - }, page.searchTerms); - }); + const searchData = { + dictionary: Array.from(dictionary.keys()).join(' '), + pages: filteredDocs.map((page) => { + // Copy the properties from the searchTerms object onto the search data object + const searchObj = { + path: page.path, + title: page.searchTitle, + type: page.docType, + }; + if (page.deprecated) { + searchObj.deprecated = true; + } + return Object.assign(searchObj, page.searchTerms); + }), + }; docs.push({ docType: 'json-doc', @@ -140,19 +108,86 @@ module.exports = function generateKeywordsProcessor(log, readFilesProcessor) { path: this.outputFolder + '/search-data.json', outputPath: this.outputFolder + '/search-data.json', data: searchData, - renderedContent: JSON.stringify(searchData) + renderedContent: JSON.stringify(searchData), }); - } + + return docs; + + // Helpers + function tokenize(text, ignoreWords, dictionary) { + // Split on whitespace and things that are likely to be HTML tags (this is not exhaustive but reduces the unwanted tokens that are indexed). + const rawTokens = text.split( + new RegExp( + '[\\s/]+' + // whitespace + '|' + // or + '', // simple HTML tags (e.g. ,
, , etc.) + 'ig' + ) + ); + const tokens = []; + for (let token of rawTokens) { + token = token.trim(); + + // Trim unwanted trivia characters from the start and end of the token + const TRIVIA_CHARS = '[\\s_"\'`({[<$*)}\\]>.,-]'; + // Tokens can contain letters, numbers, underscore, dot or hyphen but not at the start or end. + // The leading TRIVIA_CHARS will capture any leading `.`, '-`' or `_` so we don't have to avoid them in this regular expression. + // But we do need to ensure we don't capture the at the end of the token. + const POSSIBLE_TOKEN = '[a-z0-9_.-]*[a-z0-9]'; + token = token.replace(new RegExp(`^${TRIVIA_CHARS}*(${POSSIBLE_TOKEN})${TRIVIA_CHARS}*$`, 'i'), '$1'); + + // Skip if blank or in the ignored words list + if (token === '' || ignoreWords.has(token.toLowerCase())) { + continue; + } + + // Skip tokens that contain weird characters + if (!/^\w[\w.-]*$/.test(token)) { + continue; + } + + storeToken(token, tokens, dictionary); + if (token.startsWith('ng')) { + // Strip off `ng`, `ng-`, `ng1`, `ng2`, etc + storeToken(token.replace(/^ng[-12]*/, ''), tokens, dictionary); + } + } + + return tokens; + } + + function storeToken(token, tokens, dictionary) { + token = stem(token); + if (!dictionary.has(token)) { + dictionary.set(token, dictionary.size); + } + tokens.push(dictionary.get(token)); + } + + function extractMemberTokens(doc, dictionary) { + if (!doc) return []; + + let memberContent = []; + + if (doc.members) { + doc.members.forEach((member) => memberContent.push(...tokenize(member.name, emptySet, dictionary))); + } + if (doc.statics) { + doc.statics.forEach((member) => memberContent.push(...tokenize(member.name, emptySet, dictionary))); + } + if (doc.extendsClauses) { + doc.extendsClauses.forEach((clause) => memberContent.push(...extractMemberTokens(clause.doc, dictionary))); + } + if (doc.implementsClauses) { + doc.implementsClauses.forEach((clause) => memberContent.push(...extractMemberTokens(clause.doc, dictionary))); + } + + return memberContent; + } + }, }; }; - function isString(value) { return typeof value == 'string'; } - -function convertToMap(collection) { - const obj = {}; - collection.forEach(key => { obj[key] = true; }); - return obj; -} \ No newline at end of file diff --git a/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.spec.js b/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.spec.js index 7b899643d0..d35d55d412 100644 --- a/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.spec.js +++ b/docs_app/tools/transforms/angular-base-package/processors/generateKeywords.spec.js @@ -1,54 +1,109 @@ +const path = require('canonical-path'); +const Dgeni = require('dgeni'); + const testPackage = require('../../helpers/test-package'); const mockLogger = require('dgeni/lib/mocks/log')(false); const processorFactory = require('./generateKeywords'); -const Dgeni = require('dgeni'); const mockReadFilesProcessor = { - basePath: 'base/path' + basePath: 'base/path', }; -describe('generateKeywords processor', () => { +const ignoreWords = require(path.resolve(__dirname, '../ignore-words')); + +function createProcessor() { + const processor = processorFactory(mockLogger, mockReadFilesProcessor); + processor.ignoreWords = ignoreWords; + return processor; +} +describe('generateKeywords processor', () => { it('should be available on the injector', () => { - const dgeni = new Dgeni([testPackage('angular-base-package')]); + const dgeni = new Dgeni([testPackage('angular-base-package'), testPackage('angular-api-package')]); const injector = dgeni.configureInjector(); const processor = injector.get('generateKeywordsProcessor'); expect(processor.$process).toBeDefined(); }); it('should run after the correct processor', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); + const processor = createProcessor(); expect(processor.$runAfter).toEqual(['postProcessHtml']); }); it('should run before the correct processor', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); + const processor = createProcessor(); expect(processor.$runBefore).toEqual(['writing-files']); }); - it('should ignore internal and private exports', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ + it('should ignore internal and private exports', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ { docType: 'class', name: 'PublicExport' }, { docType: 'class', name: 'PrivateExport', privateExport: true }, - { docType: 'class', name: 'InternalExport', internal: true } - ]; - processor.$process(docs); - expect(docs[docs.length - 1].data).toEqual([ - jasmine.objectContaining({ title: 'PublicExport', type: 'class'}) + { docType: 'class', name: 'InternalExport', internal: true }, + ]); + expect(docs[docs.length - 1].data.pages).toEqual([jasmine.objectContaining({ title: 'PublicExport', type: 'class' })]); + }); + + it('should ignore docs that are in the `docTypesToIgnore` list', async () => { + const processor = createProcessor(); + processor.docTypesToIgnore = ['interface']; + const docs = await processor.$process([ + { docType: 'class', name: 'Class' }, + { docType: 'interface', name: 'Interface' }, + { docType: 'content', name: 'Guide' }, + ]); + expect(docs[docs.length - 1].data.pages).toEqual([ + jasmine.objectContaining({ title: 'Class', type: 'class' }), + jasmine.objectContaining({ title: 'Guide', type: 'content' }), ]); }); - it('should compute `doc.searchTitle` from the doc properties if not already provided', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ + it('should not collect keywords from properties that are in the `propertiesToIgnore` list', async () => { + const processor = createProcessor(); + processor.propertiesToIgnore = ['docType', 'ignore']; + const docs = await processor.$process([ + { docType: 'class', name: 'FooClass', ignore: 'ignore this content' }, + { docType: 'interface', name: 'BarInterface', capture: 'capture this content' }, + ]); + expect(docs[docs.length - 1].data).toEqual({ + dictionary: 'fooclass barinterfac captur content', + pages: [ + jasmine.objectContaining({ title: 'FooClass', type: 'class', keywords: [0] }), + jasmine.objectContaining({ title: 'BarInterface', type: 'interface', keywords: [1, 2, 3] }), + ], + }); + }); + + it('should not collect keywords that look like HTML tags', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ + { + docType: 'class', + name: 'FooClass', + content: ` + + + + +
Content inside a table
`, + }, + ]); + expect(docs[docs.length - 1].data).toEqual({ + dictionary: 'class fooclass content insid tabl', + pages: [jasmine.objectContaining({ keywords: [0, 1, 2, 3, 4] })], + }); + }); + + it('should compute `doc.searchTitle` from the doc properties if not already provided', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ { docType: 'class', name: 'A', searchTitle: 'searchTitle A', title: 'title A', vFile: { headings: { h1: ['vFile A'] } } }, { docType: 'class', name: 'B', title: 'title B', vFile: { headings: { h1: ['vFile B'] } } }, { docType: 'class', name: 'C', vFile: { title: 'vFile C', headings: { h1: ['vFile C'] } } }, { docType: 'class', name: 'D' }, - ]; - processor.$process(docs); - expect(docs[docs.length - 1].data).toEqual([ + ]); + expect(docs[docs.length - 1].data.pages).toEqual([ jasmine.objectContaining({ title: 'searchTitle A' }), jasmine.objectContaining({ title: 'title B' }), jasmine.objectContaining({ title: 'vFile C' }), @@ -56,118 +111,223 @@ describe('generateKeywords processor', () => { ]); }); - it('should use `doc.searchTitle` as the title in the search index', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ - { docType: 'class', name: 'PublicExport', searchTitle: 'class PublicExport' }, - ]; - processor.$process(docs); + it('should use `doc.searchTitle` as the title in the search index', async () => { + const processor = createProcessor(); + const docs = await processor.$process([{ docType: 'class', name: 'PublicExport', searchTitle: 'class PublicExport' }]); const keywordsDoc = docs[docs.length - 1]; - expect(keywordsDoc.data).toEqual([ - jasmine.objectContaining({ title: 'class PublicExport', type: 'class'}) - ]); + expect(keywordsDoc.data.pages).toEqual([jasmine.objectContaining({ title: 'class PublicExport', type: 'class' })]); }); - it('should add title words to the search terms', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ + it('should add heading words to the search terms', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ { docType: 'class', name: 'PublicExport', searchTitle: 'class PublicExport', - vFile: { headings: { h2: ['heading A', 'heading B'] } } + vFile: { headings: { h2: ['Important heading', 'Secondary heading'] } }, }, - ]; - processor.$process(docs); + ]); const keywordsDoc = docs[docs.length - 1]; - expect(keywordsDoc.data[0].titleWords).toEqual('class PublicExport'); + expect(keywordsDoc.data).toEqual({ + dictionary: 'class publicexport head secondari', + pages: [jasmine.objectContaining({ headings: [2, 3, 2] })], + }); }); - it('should add heading words to the search terms', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ + it('should add member doc properties to the search terms', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ { docType: 'class', name: 'PublicExport', searchTitle: 'class PublicExport', - vFile: { headings: { h2: ['Important heading', 'Secondary heading'] } } + vFile: { headings: { h2: ['heading A'] } }, + content: 'Some content with ngClass in it.', + members: [{ name: 'instanceMethodA' }, { name: 'instancePropertyA' }, { name: 'instanceMethodB' }, { name: 'instancePropertyB' }], + statics: [{ name: 'staticMethodA' }, { name: 'staticPropertyA' }, { name: 'staticMethodB' }, { name: 'staticPropertyB' }], }, - ]; - processor.$process(docs); + ]); const keywordsDoc = docs[docs.length - 1]; - expect(keywordsDoc.data[0].headingWords).toEqual('heading important secondary'); + expect(keywordsDoc.data).toEqual({ + dictionary: + 'class publicexport content ngclass instancemethoda instancepropertya instancemethodb instancepropertyb staticmethoda staticpropertya staticmethodb staticpropertyb head', + pages: [ + jasmine.objectContaining({ + members: [4, 5, 6, 7, 8, 9, 10, 11], + }), + ], + }); }); - it('should add member doc properties to the search terms', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ + it('should add member doc properties contained in the ignored word list to the search terms', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ { docType: 'class', name: 'PublicExport', searchTitle: 'class PublicExport', vFile: { headings: { h2: ['heading A'] } }, content: 'Some content with ngClass in it.', - members: [ - { name: 'instanceMethodA' }, - { name: 'instancePropertyA' }, - { name: 'instanceMethodB' }, - { name: 'instancePropertyB' }, - ], - statics: [ - { name: 'staticMethodA' }, - { name: 'staticPropertyA' }, - { name: 'staticMethodB' }, - { name: 'staticPropertyB' }, - ], + members: [{ name: 'some' }, { name: 'none' }, { name: 'get' }, { name: 'put' }], + statics: [{ name: 'zero' }, { name: 'one' }, { name: 'next' }, { name: 'index' }], }, - ]; - processor.$process(docs); + ]); + const keywordsDoc = docs[docs.length - 1]; + expect(keywordsDoc.data).toEqual({ + dictionary: 'class publicexport content ngclass some none get put zero on next index head', + pages: [ + jasmine.objectContaining({ + members: [4, 5, 6, 7, 8, 9, 10, 11], + }), + ], + }); + }); + + it('should add inherited member doc properties to the search terms', async () => { + const processor = createProcessor(); + const parentClass = { + docType: 'class', + name: 'ParentClass', + members: [{ name: 'parentMember1' }], + statics: [{ name: 'parentMember2' }], + }; + const parentInterface = { + docType: 'interface', + name: 'ParentInterface', + members: [{ name: 'parentMember3' }], + }; + + const childClass = { + docType: 'class', + name: 'Child', + members: [{ name: 'childMember1' }], + statics: [{ name: 'childMember2' }], + extendsClauses: [{ doc: parentClass }], + implementsClauses: [{ doc: parentInterface }], + }; + const docs = await processor.$process([childClass, parentClass, parentInterface]); const keywordsDoc = docs[docs.length - 1]; - expect(keywordsDoc.data[0].members).toEqual( - 'instancemethoda instancemethodb instancepropertya instancepropertyb staticmethoda staticmethodb staticpropertya staticpropertyb' - ); + expect(keywordsDoc.data).toEqual({ + dictionary: 'class child childmember1 childmember2 parentmember1 parentmember2 parentmember3 parentclass interfac parentinterfac', + pages: [ + jasmine.objectContaining({ + title: 'Child', + members: [2, 3, 4, 5, 6], + }), + jasmine.objectContaining({ + title: 'ParentClass', + members: [4, 5], + }), + jasmine.objectContaining({ + title: 'ParentInterface', + members: [6], + }), + ], + }); }); - it('should process terms prefixed with "ng" to include the term stripped of "ng"', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ + it('should add inherited member doc properties contained in the ignored word list to the search terms', async () => { + const processor = createProcessor(); + const parentClass = { + docType: 'class', + name: 'ParentClass', + members: [{ name: 'one' }], + statics: [{ name: 'zero' }], + }; + const parentInterface = { + docType: 'interface', + name: 'ParentInterface', + members: [{ name: 'index' }], + }; + + const childClass = { + docType: 'class', + name: 'Child', + members: [{ name: 'next' }], + statics: [{ name: 'get' }], + extendsClauses: [{ doc: parentClass }], + implementsClauses: [{ doc: parentInterface }], + }; + const docs = await processor.$process([childClass, parentClass, parentInterface]); + const keywordsDoc = docs[docs.length - 1]; + expect(keywordsDoc.data).toEqual({ + dictionary: 'class child next get on zero index parentclass interfac parentinterfac', + pages: [ + jasmine.objectContaining({ + title: 'Child', + members: [2, 3, 4, 5, 6], + }), + jasmine.objectContaining({ + title: 'ParentClass', + members: [4, 5], + }), + jasmine.objectContaining({ + title: 'ParentInterface', + members: [6], + }), + ], + }); + }); + + it('should include both stripped and unstripped "ng" prefixed tokens', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ { docType: 'class', name: 'PublicExport', searchTitle: 'ngController', vFile: { headings: { h2: ['ngModel'] } }, - content: 'Some content with ngClass in it.' + content: 'Some content with ngClass in it.', }, - ]; - processor.$process(docs); + ]); const keywordsDoc = docs[docs.length - 1]; - expect(keywordsDoc.data[0].titleWords).toEqual('ngController Controller'); - expect(keywordsDoc.data[0].headingWords).toEqual('model ngmodel'); - expect(keywordsDoc.data[0].keywords).toContain('class'); - expect(keywordsDoc.data[0].keywords).toContain('ngclass'); + expect(keywordsDoc.data).toEqual({ + dictionary: 'class publicexport ngcontrol control content ngclass ngmodel model', + pages: [ + jasmine.objectContaining({ + headings: [6, 7], + keywords: [0, 1, 2, 3, 4, 5, 0], + }), + ], + }); }); - it('should generate renderedContent property', () => { - const processor = processorFactory(mockLogger, mockReadFilesProcessor); - const docs = [ + it('should generate compressed encoded renderedContent property', async () => { + const processor = createProcessor(); + const docs = await processor.$process([ { docType: 'class', name: 'SomeClass', description: 'The is the documentation for the SomeClass API.', - vFile: { headings: { h1: ['SomeClass'], h2: ['Some heading'] } } + vFile: { headings: { h1: ['SomeClass'], h2: ['Some heading'] } }, }, - ]; - processor.$process(docs); + { + docType: 'class', + name: 'SomeClass2', + description: 'description', + members: [{ name: 'member1' }], + deprecated: true, + }, + ]); const keywordsDoc = docs[docs.length - 1]; - expect(JSON.parse(keywordsDoc.renderedContent)).toEqual( - [{ - 'title':'SomeClass', - 'type':'class', - 'titleWords':'SomeClass', - 'headingWords':'heading some someclass', - 'keywords':'api class documentation for is someclass the', - 'members':'' - }] - ); + expect(JSON.parse(keywordsDoc.renderedContent)).toEqual({ + dictionary: 'class someclass document api head someclass2 descript member1', + pages: [ + { + title: 'SomeClass', + type: 'class', + headings: [1, 4], + keywords: [0, 1, 2, 1, 3], + }, + { + title: 'SomeClass2', + type: 'class', + keywords: [0, 5, 6], + members: [7], + deprecated: true, + }, + ], + }); }); }); diff --git a/docs_app/tsconfig.worker.json b/docs_app/tsconfig.worker.json new file mode 100644 index 0000000000..54c2991361 --- /dev/null +++ b/docs_app/tsconfig.worker.json @@ -0,0 +1,16 @@ +{ + "extends": "./tsconfig.json", + "compilerOptions": { + "outDir": "./out-tsc/worker", + "types": [ + "lunr" + ], + "lib": [ + "es2018", + "webworker" + ] + }, + "include": [ + "src/**/*.worker.ts" + ] +}