Codechecks: multi-report and failure threshold options (#223)

* Make check name configurable in codechecks.yml * Add codechecks failure threshold options * Update docs
cgewecke · Oct 13, 2020 · 2732cb2 · 2732cb2
1 parent bbf521f
commit 2732cb2
Show file tree

Hide file tree

Showing 5 changed files with 102 additions and 24 deletions.
diff --git a/README.md b/README.md
@@ -93,6 +93,8 @@ your own API key [here][55] and set it with the `coinmarketcap` option.
 | proxyResolver     | _Function_             | none                        | Custom method to resolve identity of methods managed by a proxy contract.                                                                                                                                                                    |
 | artifactType      | _Function_ or _String_ | "truffle-v5"                | Compilation artifact format to consume. (See [advanced use](https://github.com/cgewecke/eth-gas-reporter/blob/master/docs/advanced.md).)                                                                                                     |
 | showMethodSig     | _Boolean_              | false                       | Display complete method signatures. Useful when you have overloaded methods you can't tell apart.                                                                                                                                            |
+| maxMethodDiff     | _Number_               | undefined                   | Codechecks failure threshold, triggered when the % diff for any method is greater than `number` (integer)                                                                                                                                    |
+| maxDeploymentDiff | _Number_               | undefined                   | Codechecks failure threshold, triggered when the % diff for any deployment is greater than `number` (integer)                                                                                                                                |
 
 [55]: https://coinmarketcap.com/api/pricing/
 

diff --git a/codechecks.js b/codechecks.js
@@ -35,23 +35,23 @@ module.exports.default = async function gasReporter(options = {}) {
     ? `${output.namespace}:${options.name}`
     : output.namespace;
 
-  // Save new data on the merge commit / push build
-  if (!codechecks.isPr()) {
-    const report = new CodeChecksReport(output.config);
-    report.generate(output.info);
+  let report = new CodeChecksReport(output.config);
+  report.generate(output.info);
 
-    try {
-      await codechecks.saveValue(output.namespace, report.newData);
-      console.log(`Successful save: output.namespace was: ${output.namespace}`);
-    } catch (err) {
-      console.log(
-        `If you have a chance, report this incident to the eth-gas-reporter github issues.`
-      );
-      console.log(`Codechecks errored running 'saveValue'...\n${err}\n`);
-      console.log(`output.namespace was: ${output.namespace}`);
-      console.log(`Saved gas-reporter data was: ${report.newData}`);
-    }
+  try {
+    await codechecks.saveValue(output.namespace, report.newData);
+    console.log(`Successful save: output.namespace was: ${output.namespace}`);
+  } catch (err) {
+    console.log(
+      `If you have a chance, report this incident to the eth-gas-reporter github issues.`
+    );
+    console.log(`Codechecks errored running 'saveValue'...\n${err}\n`);
+    console.log(`output.namespace was: ${output.namespace}`);
+    console.log(`Saved gas-reporter data was: ${report.newData}`);
+  }
 
+  // Exit early on merge commit / push build
+  if (!codechecks.isPr()) {
     return;
   }
 
@@ -66,22 +66,29 @@ module.exports.default = async function gasReporter(options = {}) {
     return;
   }
 
-  const report = new CodeChecksReport(output.config);
+  report = new CodeChecksReport(output.config);
   const table = report.generate(output.info);
   const shortDescription = report.getShortDescription();
 
+  // Support multiple reports
+  const checkName = options.name ? `Gas Usage: ${options.name}` : `Gas Usage`;
+
   // Submit report
   try {
-    await codechecks.success({
-      name: "Gas Usage",
+    const payload = {
+      name: checkName,
       shortDescription: shortDescription,
       longDescription: table
-    });
+    };
+
+    report.success
+      ? await codechecks.success(payload)
+      : await codechecks.failure(payload);
   } catch (err) {
     console.log(
       `If you have a chance, report this incident to the eth-gas-reporter github issues.`
     );
-    console.log(`Codechecks errored running 'success'...\n${err}\n`);
+    console.log(`Codechecks errored running .success or .failure\n${err}\n`);
     console.log(`Short description was: ${shortDescription}`);
     console.log(`Table was: ${table}`);
   }

diff --git a/docs/codechecks.md b/docs/codechecks.md
@@ -44,6 +44,61 @@ script:
 
 - You're done! :elephant:
 
+### Multiple reports (for different CI jobs)
+
+For each report, create a codechecks.yml file, e.g
+
+```
+codechecks.testing.yml
+codechecks.production.yml
+```
+
+Use the `name` option in your `.yml` config to individuate the report:
+
+```yml
+# codechecks.production.yml
+checks:
+  - name: eth-gas-reporter/codechecks
+    options:
+      name: production
+```
+
+When running `codechecks` as a command in CI, specify the relevant codechecks config `.yml`
+
+```yml
+production:
+  docker:
+    - image: circleci/node:10.13.0
+  steps:
+    - checkout
+    - run: npm install
+    - run: npm test
+    - run: npx codechecks codechecks.production.yml
+```
+
+### Failure thresholds
+
+You can ask Codechecks to report the CI run as a failure by using the `maxMethodDiff` and
+`maxDeploymentDiff` reporter options. These set the integer percentage difference
+over which an increase in gas usage by any method (or deployment) is forbidden.
+
+**Examples**
+
+```js
+// truffle-config.js
+mocha: {
+  reporter: "eth-gas-reporter",
+  reporterOptions: {
+    maxMethodDiff: 25,
+  }
+}
+
+// buidler.config.js
+gasReporter: {
+  maxMethodDiff: 25,
+}
+```
+
 ### Codechecks is new :wrench:
 
 Codechecks is new and some of its quirks are still being ironed out:

diff --git a/lib/codechecksReport.js b/lib/codechecksReport.js
@@ -11,6 +11,7 @@ class CodeChecksReport {
     this.increases = 0;
     this.decreases = 0;
     this.reportIsNew = true;
+    this.success = true;
 
     this.previousData = config.previousData || { methods: {}, deployments: {} };
     this.newData = { methods: {}, deployments: {} };
@@ -239,13 +240,19 @@ class CodeChecksReport {
     return "-";
   }
 
-  getPercentageDiff(previousVal, currentVal) {
+  getPercentageDiff(previousVal, currentVal, maxThreshold) {
     let sign = "";
 
     if (typeof previousVal === "number") {
       const diff = Math.round(((currentVal - previousVal) / previousVal) * 100);
 
-      if (diff > 0) sign = "+";
+      if (diff > 0) {
+        sign = "+";
+
+        if (typeof maxThreshold === "number" && diff > maxThreshold) {
+          this.success = false;
+        }
+      }
 
       return `${sign}${diff}%`;
     }
@@ -257,7 +264,11 @@ class CodeChecksReport {
   }
 
   getMethodPercentageDiff(id, currentVal) {
-    return this.getPercentageDiff(this.previousData.methods[id], currentVal);
+    return this.getPercentageDiff(
+      this.previousData.methods[id],
+      currentVal,
+      this.config.maxMethodDiff
+    );
   }
 
   getDeploymentDiff(id, currentVal) {
@@ -267,7 +278,8 @@ class CodeChecksReport {
   getDeploymentPercentageDiff(id, currentVal) {
     return this.getPercentageDiff(
       this.previousData.deployments[id],
-      currentVal
+      currentVal,
+      this.config.maxDeploymentDiff
     );
   }
 

diff --git a/lib/config.js b/lib/config.js
@@ -23,6 +23,8 @@ class Config {
     this.metadata = options.metadata || null;
     this.showMethodSig = options.showMethodSig || false;
     this.provider = options.provider || null;
+    this.maxMethodDiff = options.maxMethodDiff;
+    this.maxDeploymentDiff = options.maxDeploymentDiff;
 
     this.excludeContracts = Array.isArray(options.excludeContracts)
       ? options.excludeContracts