Generate super referral mapping table component

package.json

@@ -40,6 +40,7 @@
     "data-files-speedreader": "npm run --prefix ./node_modules/speedreader data-files",
     "generate-ntp-sponsored-images": "node scripts/generateNTPSponsoredImages.js",
     "generate-ntp-super-referrer": "node scripts/generateNTPSuperReferrer.js",
+    "generate-ntp-super-referrer-mapping-table": "node scripts/generateNTPSuperReferrerMappingTable.js",
     "generate-parameter-updates": "node scripts/generateClientModelParameterFiles.js",
     "lint": "standard",
     "import-cws-components": "node ./scripts/importCWSComponents",
@@ -51,6 +52,7 @@
     "package-local-data-files": "node ./scripts/packageComponent --type local-data-files-updater",
     "package-ntp-sponsored-images": "node ./scripts/packageNTPSponsoredImagesComponents",
     "package-ntp-super-referrer": "node scripts/packageNTPSuperReferrerComponent.js",
+    "package-ntp-super-referrer-mapping-table": "node scripts/packageNTPSuperReferrerMappingTableComponent.js",
     "package-speedreader": "node ./scripts/packageComponent --type speedreader-updater",
     "upload-ethereum-remote-client": "node ./scripts/uploadComponent --type ethereum-remote-client",
     "package-parameter-updates": "node ./scripts/packageClientModelParameterComponent",
@@ -62,6 +64,7 @@
     "upload-local-data-files": "node ./scripts/uploadComponent --type local-data-files-updater",
     "upload-ntp-sponsored-images-components": "node ./scripts/uploadComponent --type ntp-sponsored-images",
     "upload-ntp-super-referrer-component": "node ./scripts/uploadComponent --type ntp-super-referrer",
+    "upload-ntp-super-referrer-component-mapping-table": "node ./scripts/uploadComponent --type ntp-super-referrer-mapping-table",
     "upload-parameter-updates": "node ./scripts/uploadComponent --type client-model-parameters",
     "upload-speedreader": "node ./scripts/uploadComponent --type speedreader-updater"
   },

scripts/generateNTPSuperReferrerMappingTable.js

@@ -0,0 +1,64 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const path = require('path')
+const mkdirp = require('mkdirp')
+const fs = require('fs-extra')
+const request = require('request')
+const commander = require('commander')
+
+const jsonSchemaVersion = 1
+
+const createDataJsonFile = (path, body) => {
+  fs.writeFileSync(path, body)
+}
+
+function downloadMappingTableJsonFile (jsonFileUrl, targetFilePath) {
+  return new Promise(function (resolve, reject) {
+    let jsonFileBody = '{}'
+
+    request(jsonFileUrl, async function (error, response, body) {
+      if (error) {
+        console.error(`Error from ${jsonFileUrl}:`, error)
+        return reject(error)
+      }
+      if (response && response.statusCode === 200) {
+        jsonFileBody = body
+      }
+
+      const data = JSON.parse(jsonFileBody)
+      // Make sure the data has a schema version so that clients can opt to parse or not
+      const incomingSchemaVersion = data.schemaVersion
+      if (!incomingSchemaVersion) {
+        // Source has no schema version, assume and set current version.
+        // TODO(petemill): Don't allow this once the source is established to always
+        // have a schema version.
+        data.schemaVersion = jsonSchemaVersion
+      } else if (incomingSchemaVersion !== jsonSchemaVersion) {
+        // We don't support this file format
+        console.error(`Error: Cannot parse JSON data at ${jsonFileUrl} since it has a schema version of ${incomingSchemaVersion} but we expected ${jsonSchemaVersion}! This region will not be updated.`)
+        return reject(error)
+      }
+
+      createDataJsonFile(targetFilePath, JSON.stringify(data))
+
+      resolve()
+    })
+  })
+}
+
+async function generateNTPSuperReferrerMappingTable (dataUrl) {
+  const rootResourceDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table')
+  mkdirp.sync(rootResourceDir)
+
+  console.log(`Downloading ${dataUrl}...`)
+  const targetFilePath = path.join(rootResourceDir, 'mapping-table.json')
+  await downloadMappingTableJsonFile(dataUrl, targetFilePath)
+}
+
+commander
+  .option('-d, --data-url <url>', 'url that refers to data that has ntp super referrer')
+  .parse(process.argv)
+
+generateNTPSuperReferrerMappingTable(commander.dataUrl)

scripts/packageNTPSuperReferrerMappingTableComponent.js

@@ -0,0 +1,110 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this file,
+ * You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+const childProcess = require('child_process')
+const commander = require('commander')
+const fs = require('fs-extra')
+const mkdirp = require('mkdirp')
+const path = require('path')
+const replace = require('replace-in-file')
+const util = require('../lib/util')
+
+const stageFiles = (version, outputDir) => {
+  // Copy mapping table and manifest file to outputDir.
+  const mapFile = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'resources', 'mapping-table', 'mapping-table.json')
+  const outputMapFile = path.join(outputDir, 'mapping-table.json')
+  console.log('copy ', mapFile, ' to:', outputMapFile)
+  fs.copyFileSync(mapFile, outputMapFile)
+
+  // Fix up the manifest version
+  const originalManifest = getOriginalManifest()
+  const outputManifest = path.join(outputDir, 'manifest.json')
+  console.log('copy manifest file: ', originalManifest, ' to: ', outputManifest)
+  const replaceOptions = {
+    files: outputManifest,
+    from: /0\.0\.0/,
+    to: version
+  }
+  fs.copyFileSync(originalManifest, outputManifest)
+  replace.sync(replaceOptions)
+}
+
+const generateManifestFile = (publicKey) => {
+  const manifestFile = getOriginalManifest()
+  const manifestContent = {
+    description: 'Brave NTP Super Referrer mapping table component',
+    key: publicKey,
+    manifest_version: 2,
+    name: `Brave NTP Super Referrer mapping table`,
+    version: '0.0.0'
+  }
+  fs.writeFileSync(manifestFile, JSON.stringify(manifestContent))
+}
+
+const getOriginalManifest = () => {
+  return path.join(path.resolve(), 'build','ntp-super-referrer', 'mapping-table-manifest.json')
+}
+
+const generatePublicKeyAndID = (privateKeyFile) => {
+  childProcess.execSync(`openssl rsa -in ${privateKeyFile} -pubout -out public.pub`)
+  try {
+      // read contents of the file
+      const data = fs.readFileSync('public.pub', 'UTF-8');
+
+      // split the contents by new line
+      const lines = data.split(/\r?\n/);
+      let pubKeyString = ''
+      lines.forEach((line) => {
+          if (!line.includes('-----'))
+            pubKeyString += line
+      });
+      console.log(`publicKey: ${pubKeyString}`)
+      const id = util.getIDFromBase64PublicKey(pubKeyString)
+      console.log(`componentID: ${id}`)
+      return [pubKeyString, id]
+  } catch (err) {
+      console.error(err);
+  }
+}
+
+const generateCRXFile = (binary, endpoint, region, componentID, privateKeyFile) => {
+  const originalManifest = getOriginalManifest()
+  const rootBuildDir = path.join(path.resolve(), 'build', 'ntp-super-referrer', 'mapping-table')
+  const stagingDir = path.join(rootBuildDir, 'staging')
+  const crxOutputDir = path.join(rootBuildDir, 'output')
+  mkdirp.sync(stagingDir)
+  mkdirp.sync(crxOutputDir)
+  util.getNextVersion(endpoint, region, componentID).then((version) => {
+    const crxFile = path.join(crxOutputDir, `ntp-super-referrer-mapping-table.crx`)
+    stageFiles(version, stagingDir)
+    util.generateCRXFile(binary, crxFile, privateKeyFile, stagingDir)
+    console.log(`Generated ${crxFile} with version number ${version}`)
+  })
+}
+
+util.installErrorHandlers()
+
+commander
+  .option('-b, --binary <binary>', 'Path to the Chromium based executable to use to generate the CRX file')
+  .option('-k, --key <file>', 'file containing private key for signing crx file')
+  .option('-e, --endpoint <endpoint>', 'DynamoDB endpoint to connect to', '')// If setup locally, use http://localhost:8000
+  .option('-r, --region <region>', 'The AWS region to use', 'us-east-2')
+  .parse(process.argv)
+
+let privateKeyFile = ''
+if (fs.existsSync(commander.key)) {
+  privateKeyFile = commander.key
+} else {
+  throw new Error('Missing or invalid private key')
+}
+
+if (!commander.binary) {
+  throw new Error('Missing Chromium binary: --binary')
+}
+
+util.createTableIfNotExists(commander.endpoint, commander.region).then(() => {
+  const [publicKey, componentID] = generatePublicKeyAndID(privateKeyFile)
+  generateManifestFile(publicKey)
+  generateCRXFile(commander.binary, commander.endpoint, commander.region, componentID, privateKeyFile)
+})

scripts/uploadComponent.js

@@ -12,7 +12,7 @@ util.installErrorHandlers()
 commander
   .option('-d, --crx-directory <dir>', 'directory containing multiple crx files to upload')
   .option('-f, --crx-file <file>', 'crx file to upload', 'extension.crx')
-  .option('-t, --type <type>', 'component extension type', /^(ad-block-updater|https-everywhere-updater|local-data-files-updater|ethereum-remote-client|ntp-sponsored-images|ntp-super-referrer|speedreader-updater|tor-client-updater|client-model-parameters)$/i, 'ad-block-updater')
+  .option('-t, --type <type>', 'component extension type', /^(ad-block-updater|https-everywhere-updater|local-data-files-updater|ethereum-remote-client|ntp-sponsored-images|ntp-super-referrer|ntp-super-referrer-mapping-table|speedreader-updater|tor-client-updater|client-model-parameters)$/i, 'ad-block-updater')
   .option('-e, --endpoint <endpoint>', 'DynamoDB endpoint to connect to', '')// If setup locally, use http://localhost:8000
   .option('-r, --region <region>', 'The AWS region to use', 'us-east-2')
   .parse(process.argv)