initial implementation of an executable server

The goal of this PR is to setup a browser version of Blueprint packaged as a "drag and drop" that's easy to deploy. The setup is a bit tricky and might not apply well to every kind of product made using the Theia framework, but it should work for most usages. Add `theia-blueprint-browser` Theia extension that contributes the required customizations to run in the packaged environment (mostly to accommodate to the new file layout). Use a `node`-specific Webpack configuration to bundle the backend along with the browser-specific customizations. Use a forked `pkg` package to support forking child processes with `execArgv` options. The fork is setup as a git submodule.
eclipse-theia · Oct 13, 2021 · d5db3df · d5db3df
1 parent abfbcd5
commit d5db3df
Show file tree

Hide file tree

Showing 23 changed files with 1,769 additions and 189 deletions.
diff --git a/.gitignore b/.gitignore
@@ -1,10 +1,10 @@
 .DS_Store
-**/node_modules
-**/.browser_modules
-**/dist
-**/lib
-**/src-gen
-**/gen-webpack.config.js
-**/plugins
-**/tsconfig.tsbuildinfo
-*.log
+node_modules
+.browser_modules
+dist
+lib
+src-gen
+gen-webpack.config.js
+plugins
+tsconfig.tsbuildinfo
+*.log
diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,4 @@
+[submodule "submodules/pkg"]
+	path = submodules/pkg
+	url = https://github.com/marechal-p/pkg.git
+	branch = exec-argv
diff --git a/applications/browser/.gitignore b/applications/browser/.gitignore
@@ -0,0 +1,3 @@
+/bundled
+/builtins
+/packaged
diff --git a/applications/browser/node-webpack.config.js b/applications/browser/node-webpack.config.js
@@ -0,0 +1,114 @@
+const path = require('path');
+const webpack = require('webpack');
+const CopyPlugin = require('copy-webpack-plugin');
+const TerserPlugin = require('terser-webpack-plugin');
+
+/** @type {import('webpack').Configuration['mode']} */
+const mode = 'production';
+
+/** @type {import('webpack').EntryObject} */
+const commonJsLibraries = {};
+for (const [entryPointName, entryPointPath] of Object.entries({
+    'backend-init-theia': '@theia/plugin-ext/lib/hosted/node/scanners/backend-init-theia',
+    'git-locator-host': '@theia/git/lib/node/git-locator/git-locator-host',
+    'nsfw-watcher': '@theia/filesystem/lib/node/nsfw-watcher',
+    'plugin-vscode-init': '@theia/plugin-ext-vscode/lib/node/plugin-vscode-init',
+})) {
+    commonJsLibraries[entryPointName] = {
+        import: require.resolve(entryPointPath),
+        library: {
+            type: 'commonjs2',
+        },
+    };
+}
+
+const ignoreResources = new Set([
+    'node-ssh',
+    'vertx'
+])
+
+/** @type {import('webpack').Configuration} */
+module.exports = {
+    mode,
+    devtool: mode === 'development' ? 'source-map' : false,
+    target: 'node',
+    node: {
+        global: false,
+        __filename: false,
+        __dirname: false
+    },
+    output: {
+        filename: '[name].js',
+        path: path.resolve(__dirname, 'bundled')
+    },
+    entry: {
+        // Main entry point of the Theia application backend:
+        'blueprint': require.resolve('./src-gen/backend/main'),
+        // Theia's IPC mechanism:
+        'ipc-bootstrap': require.resolve('@theia/core/lib/node/messaging/ipc-bootstrap'),
+        // VS Code extension support:
+        'plugin-host': require.resolve('@theia/plugin-ext/lib/hosted/node/plugin-host'),
+        ...commonJsLibraries
+    },
+    externals: {
+        child_process: 'commonjs2 child_process'
+    },
+    module: {
+        parser: {
+            javascript: {
+                // `@theia/core` `dynamicRequire` function requires Webpack's magic comments
+                // to allow dynamic requires. Otherwise Webpack will replace any dynamic require
+                // call by a hardcoded function that throws an error when invoked.
+                commonjsMagicComments: true,
+            },
+        },
+        rules: [
+            // Make sure we can still find and load our native addons.
+            {
+                test: /\.node$/,
+                loader: 'node-loader',
+                options: {
+                    name: 'native/[name].[ext]'
+                }
+            },
+            // jsonc-parser exposes its UMD implementation by default, which
+            // confuses Webpack leading to missing js in the bundles.
+            {
+                test: /node_modules[\\/](jsonc-parser)/,
+                loader: 'umd-compat-loader'
+            }
+        ]
+    },
+    plugins: [
+        new webpack.NormalModuleReplacementPlugin(/^bindings$/, path.resolve('replacements/bindings.js')),
+        // Webpack trips on the places where those modules are required.
+        // Since we'll never reach the code paths where they actually are required at runtime,
+        // it is safe to completely ignore them. Webpack will throw an error if they are required.
+        new webpack.IgnorePlugin({
+            checkResource: (resource) => ignoreResources.has(resource)
+        }),
+        new CopyPlugin({
+            patterns: [
+                {
+                    // Copy over ripgrep's binaries
+                    context: path.resolve(require.resolve('vscode-ripgrep/package.json'), '../bin'),
+                    from: '*',
+                    to: 'bin'
+                },
+            ]
+        })
+    ],
+    optimization: {
+        // Split and reuse code across the various entry points
+        splitChunks: {
+            chunks: 'all'
+        },
+        // Only minimize if we run webpack in production mode
+        minimize: mode === 'production',
+        minimizer: [
+            new TerserPlugin({
+                exclude: /^(lib|builtins)\//
+            })
+        ]
+    },
+};
diff --git a/applications/browser/package.json b/applications/browser/package.json
@@ -0,0 +1,119 @@
+{
+  "private": true,
+  "name": "theia-blueprint-node",
+  "description": "Eclipse Theia blueprint product",
+  "productName": "Theia Blueprint",
+  "version": "1.18.0",
+  "main": "scripts/theia-electron-main.js",
+  "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
+  "author": "Rob Moran <github@thegecko.org>",
+  "homepage": "https://github.com/eclipse-theia/theia-blueprint#readme",
+  "bugs": {
+    "url": "https://github.com/eclipse-theia/theia/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/eclipse-theia/theia-blueprint.git"
+  },
+  "engines": {
+    "yarn": "1.0.x || >=1.2.1"
+  },
+  "scripts": {
+    "bundle": "yarn bundle:frontend && yarn bundle:backend",
+    "bundle:frontend": "theia build --mode=production",
+    "bundle:backend": "yarn rebuild && webpack --config node-webpack.config.js",
+    "clean": "theia clean && rimraf bundled packaged",
+    "deploy": "echo skip",
+    "lint": "theiaext lint",
+    "package": "yarn package:clean && yarn package:pkg && yarn package:copy",
+    "package:clean": "rimraf packaged",
+    "package:copy": "cp -r builtins bundled/bin lib packaged",
+    "package:pkg": "pkg --public --public-packages \"*\" --no-bytecode -t node12 -c package.json -o packaged/blueprint.exe bundled/blueprint.js",
+    "prepare": "theia download:plugins",
+    "rebuild": "theia rebuild:browser --cacheRoot ../.."
+  },
+  "theia": {
+    "target": "browser",
+    "frontend": {
+      "config": {
+        "applicationName": "Theia Blueprint (Web)"
+      }
+    },
+    "backend": {
+      "config": {
+        "startupTimeout": -1
+      }
+    }
+  },
+  "pkg": {
+    "bin": "bundled/blueprint.js",
+    "outputPath": "packaged",
+    "assets": [
+      "package.json",
+      "bundled/*.js",
+      "bundled/native/*.node"
+    ]
+  },
+  "dependencies": {
+    "@theia/bulk-edit": "1.18.0",
+    "@theia/callhierarchy": "1.18.0",
+    "@theia/console": "1.18.0",
+    "@theia/core": "1.18.0",
+    "@theia/debug": "1.18.0",
+    "@theia/editor": "1.18.0",
+    "@theia/editor-preview": "1.18.0",
+    "@theia/external-terminal": "1.18.0",
+    "@theia/file-search": "1.18.0",
+    "@theia/filesystem": "1.18.0",
+    "@theia/getting-started": "1.18.0",
+    "@theia/git": "1.18.0",
+    "@theia/keymaps": "1.18.0",
+    "@theia/markers": "1.18.0",
+    "@theia/messages": "1.18.0",
+    "@theia/mini-browser": "1.18.0",
+    "@theia/monaco": "1.18.0",
+    "@theia/navigator": "1.18.0",
+    "@theia/outline-view": "1.18.0",
+    "@theia/output": "1.18.0",
+    "@theia/plugin-dev": "1.18.0",
+    "@theia/plugin-ext": "1.18.0",
+    "@theia/plugin-ext-vscode": "1.18.0",
+    "@theia/preferences": "1.18.0",
+    "@theia/preview": "1.18.0",
+    "@theia/process": "1.18.0",
+    "@theia/property-view": "1.18.0",
+    "@theia/scm": "1.18.0",
+    "@theia/scm-extra": "1.18.0",
+    "@theia/search-in-workspace": "1.18.0",
+    "@theia/task": "1.18.0",
+    "@theia/terminal": "1.18.0",
+    "@theia/timeline": "1.18.0",
+    "@theia/typehierarchy": "1.18.0",
+    "@theia/userstorage": "1.18.0",
+    "@theia/variable-resolver": "1.18.0",
+    "@theia/vsx-registry": "1.18.0",
+    "@theia/workspace": "1.18.0",
+    "theia-blueprint-product": "1.18.0",
+    "theia-blueprint-browser": "1.18.0"
+  },
+  "devDependencies": {
+    "@theia/cli": "1.18.0",
+    "node-loader": "^1.0.3",
+    "pkg": "5.1.0"
+  },
+  "theiaPluginsDir": "builtins",
+  "theiaPlugins": {
+    "vscode.git": "https://open-vsx.org/api/vscode/git/1.52.1/file/vscode.git-1.52.1.vsix",
+    "vscode.markdown-language-features": "https://open-vsx.org/api/vscode/markdown-language-features/1.39.2/file/vscode.markdown-language-features-1.39.2.vsix",
+    "vscode-builtin-extensions-pack": "https://open-vsx.org/api/eclipse-theia/builtin-extension-pack/1.50.1/file/eclipse-theia.builtin-extension-pack-1.50.1.vsix",
+    "redhat.java": "https://open-vsx.org/api/redhat/java/0.73.0/file/redhat.java-0.73.0.vsix",
+    "vscjava.vscode-java-debug": "https://open-vsx.org/api/vscjava/vscode-java-debug/0.30.0/file/vscjava.vscode-java-debug-0.30.0.vsix",
+    "vscjava.vscode-java-test": "https://open-vsx.org/api/vscjava/vscode-java-test/0.26.1/file/vscjava.vscode-java-test-0.26.1.vsix",
+    "vscjava.vscode-maven": "https://open-vsx.org/api/vscjava/vscode-maven/0.21.2/file/vscjava.vscode-maven-0.21.2.vsix",
+    "vscjava.vscode-java-dependency": "https://open-vsx.org/api/vscjava/vscode-java-dependency/0.16.0/file/vscjava.vscode-java-dependency-0.16.0.vsix"
+  },
+  "theiaPluginsExcludeIds": [
+    "vscode.extension-editing",
+    "vscode.microsoft-authentication"
+  ]
+}
diff --git a/applications/browser/replacements/bindings.js b/applications/browser/replacements/bindings.js
@@ -0,0 +1,26 @@
+/********************************************************************************
+ * Copyright (C) 2020 Ericsson and others.
+ *
+ * This program and the accompanying materials are made available under the
+ * terms of the Eclipse Public License v. 2.0 which is available at
+ * http://www.eclipse.org/legal/epl-2.0.
+ *
+ * This Source Code may also be made available under the following Secondary
+ * Licenses when the conditions for such availability set forth in the Eclipse
+ * Public License v. 2.0 are satisfied: GNU General Public License, version 2
+ * with the GNU Classpath Exception which is available at
+ * https://www.gnu.org/software/classpath/license.html.
+ *
+ * SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+ ********************************************************************************/
+
+/**
+ * @file Re-implement the `bindings` package to be Webpack-friendly.
+ */
+
+ module.exports = function (jsModule) {
+    switch (jsModule) {
+        case 'drivelist': return require('drivelist/build/Release/drivelist.node');
+    }
+    throw new Error(`unhandled module: "${jsModule}"`);
+}
diff --git a/applications/browser/webpack.config.js b/applications/browser/webpack.config.js
@@ -0,0 +1,17 @@
+/**
+ * This file can be edited to customize webpack configuration.
+ * To reset delete this file and rerun theia build again.
+ */
+// @ts-check
+const config = require('./gen-webpack.config.js');
+
+/**
+ * Expose bundled modules on window.theia.moduleName namespace, e.g.
+ * window['theia']['@theia/core/lib/common/uri'].
+ * Such syntax can be used by external code, for instance, for testing.
+config.module.rules.push({
+    test: /\.js$/,
+    loader: require.resolve('@theia/application-manager/lib/expose-loader')
+}); */
+
+module.exports = config;
diff --git a/applications/electron/package.json b/applications/electron/package.json
@@ -6,7 +6,7 @@
   "version": "1.18.0",
   "main": "scripts/theia-electron-main.js",
   "license": "EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0",
-  "author": "Eclipse Theia <theia-dev@eclipse.org>",
+  "author": "Rob Moran <github@thegecko.org>",
   "homepage": "https://github.com/eclipse-theia/theia-blueprint#readme",
   "bugs": {
     "url": "https://github.com/eclipse-theia/theia/issues"
@@ -71,13 +71,13 @@
     "@theia/variable-resolver": "1.18.0",
     "@theia/vsx-registry": "1.18.0",
     "@theia/workspace": "1.18.0",
-    "fs-extra": "^9.0.1",
     "theia-blueprint-updater": "1.18.0",
     "theia-blueprint-product": "1.18.0"
   },
   "devDependencies": {
     "@theia/cli": "1.18.0",
     "@types/js-yaml": "^3.12.0",
+    "@types/node": "12",
     "@types/yargs": "^17.0.2",
     "@wdio/cli": "^6.10.2",
     "@wdio/local-runner": "^6.10.2",
@@ -86,6 +86,7 @@
     "@wdio/sync": "^6.10.0",
     "app-builder-lib": "^22.9.0",
     "chai": "^4.2.0",
+    "electron": "9.3.2",
     "electron-builder": "^22.8.0",
     "electron-chromedriver": "9.0.0",
     "electron-mocha": "^9.3.2",
@@ -98,20 +99,20 @@
     "yargs": "^17.0.1"
   },
   "scripts": {
-    "prepare": "yarn build && yarn download:plugins",
+    "bundle": "theia build",
     "clean": "theia clean && rimraf node_modules",
     "clean:dist": "rimraf dist",
-    "build": "theia rebuild:electron && yarn bundle",
-    "bundle": "theia build",
-    "watch": "concurrently -n compile,bundle \"theiaext watch --preserveWatchOutput\" \"theia build --watch --mode development\"",
-    "start": "electron scripts/theia-electron-main.js",
-    "start:debug": "yarn start --log-level=debug",
-    "package": "yarn clean:dist && electron-builder -c.mac.identity=null --publish never",
     "deploy": "yarn clean:dist && electron-builder -c.mac.identity=null --publish always",
-    "package:preview": "yarn clean:dist && electron-builder --dir",
-    "update:checksum": "ts-node scripts/update-checksum.ts",
     "download:plugins": "theia download:plugins",
-    "test": "mocha --timeout 60000 \"./test/*.spec.js\""
+    "lint": "theiaext lint",
+    "package": "yarn clean:dist && yarn rebuild && electron-builder -c.mac.identity=null --publish never",
+    "package:preview": "yarn clean:dist && yarn rebuild && electron-builder --dir",
+    "prepare": "yarn download:plugins",
+    "rebuild": "theia rebuild:electron --cacheRoot ../..",
+    "start": "yarn rebuild && electron scripts/theia-electron-main.js",
+    "start:debug": "yarn start --log-level=debug",
+    "test": "yarn rebuild && mocha --timeout 60000 \"./test/*.spec.js\"",
+    "watch": "concurrently -n compile,bundle \"theiaext watch --preserveWatchOutput\" \"theia build --watch --mode development\""
   },
   "theiaPluginsDir": "plugins",
   "theiaPlugins": {

diff --git a/applications/electron/scripts/notarize.sh b/applications/electron/scripts/notarize.sh
diff --git a/applications/electron/scripts/sign.sh b/applications/electron/scripts/sign.sh