feature(db): cleanup (#44)

* make sites.domain index unique
* remove unused sites fields
* remove unused scans fields

migrations/008.do.unique_index_on_sites_domain.sql

@@ -0,0 +1,2 @@
+CREATE UNIQUE INDEX IF NOT EXISTS sites_domain_unique_idx ON sites (domain);
+DROP INDEX IF EXISTS sites_domain_idx;

migrations/008.undo.unique_index_on_sites_domain.sql

@@ -0,0 +1,2 @@
+DROP INDEX IF EXISTS sites_domain_unique_idx;
+CREATE INDEX IF NOT EXISTS sites_domain_idx ON sites (domain);

migrations/009.do.remove_sites_fields.sql

@@ -0,0 +1,4 @@
+ALTER TABLE sites 
+  DROP COLUMN IF EXISTS public_headers, 
+  DROP COLUMN IF EXISTS private_headers, 
+  DROP COLUMN IF EXISTS cookies;

migrations/009.undo.remove_sites_fields.sql

@@ -0,0 +1,3 @@
+ALTER TABLE sites ADD COLUMN IF NOT EXISTS public_headers   jsonb;
+ALTER TABLE sites ADD COLUMN IF NOT EXISTS private_headers  jsonb;
+ALTER TABLE sites ADD COLUMN IF NOT EXISTS cookies          jsonb;

migrations/010.do.remove_scans_fields.sql

@@ -0,0 +1,3 @@
+ALTER TABLE scans 
+  DROP COLUMN IF EXISTS "hidden", 
+  DROP COLUMN IF EXISTS likelihood_indicator;

migrations/010.undo.remove_scans_fields.sql

@@ -0,0 +1,4 @@
+ALTER TABLE scans ADD COLUMN IF NOT EXISTS hidden boolean NOT NULL DEFAULT false;
+ALTER TABLE scans ADD COLUMN IF NOT EXISTS likelihood_indicator VARCHAR NULL;
+
+CREATE INDEX IF NOT EXISTS scans_hidden_idx ON scans(hidden);

package.json

@@ -24,6 +24,7 @@
   "devDependencies": {
     "@faker-js/faker": "^8.4.1",
     "@supercharge/promise-pool": "^3.2.0",
+    "@types/mocha": "^10.0.7",
     "chai": "^5.1.1",
     "json-schema-to-jsdoc": "^1.1.1",
     "mocha": "^10.7.0",

src/database/repository.js

@@ -57,26 +57,23 @@ export const ScanState = {
  * @property {number} tests_quantity
  * @property {number | null} grade
  * @property {number | null} score
- * @property {string | null} likelihood_indicator
  * @property {string | null} error
  * @property {Object | null} response_headers
- * @property {boolean} hidden
  * @property {number | null} status_code
  */
 
 /**
  *
  * @param {Pool} pool
  * @param {number} siteId
- * @param {boolean} hidden
  * @returns {Promise<ScanRow>}
  */
-export async function insertScan(pool, siteId, hidden = false) {
+export async function insertScan(pool, siteId) {
   const result = await pool.query(
-    `INSERT INTO scans (site_id, state, start_time, tests_quantity, hidden, algorithm_version)
-      VALUES ($1, $2, NOW(), 0, $3, $4)
+    `INSERT INTO scans (site_id, state, start_time, tests_quantity, algorithm_version)
+      VALUES ($1, $2, NOW(), 0, $3)
       RETURNING *`,
-    [siteId, ScanState.RUNNING, hidden, ALGORITHM_VERSION]
+    [siteId, ScanState.RUNNING, ALGORITHM_VERSION]
   );
   /** @type {ScanRow} */
   const row = result.rows["0"];
@@ -131,17 +128,16 @@ export async function insertTestResults(pool, siteId, scanId, scanResult) {
   const scan = scanResult.scan;
   const result = await pool.query(
     `UPDATE scans
-      SET (end_time, tests_failed, tests_passed, grade, score, likelihood_indicator,
+      SET (end_time, tests_failed, tests_passed, grade, score,
       state, response_headers, status_code, algorithm_version, tests_quantity, error) =
-      (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
-      WHERE id = $12
+      (NOW(), $1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
+      WHERE id = $11
       RETURNING *`,
     [
       scan.testsFailed,
       scan.testsPassed,
       scan.grade,
       scan.score,
-      scan.likelihoodIndicator,
       scan.score !== null ? ScanState.FINISHED : ScanState.FAILED,
       scan.responseHeaders,
       scan.statusCode,

src/grader/charts.js

@@ -27,17 +27,6 @@ export const GRADE_CHART = new Map([
   [0, "F"],
 ]);
 
-/**
- * @type {Map<string, string>}
- * */
-export const LIKELIHOOD_INDICATOR_CHART = new Map([
-  ["A", "LOW"],
-  ["B", "MEDIUM"],
-  ["C", "MEDIUM"],
-  ["D", "MEDIUM"],
-  ["F", "MEDIUM"],
-]);
-
 export const MINIMUM_SCORE_FOR_EXTRA_CREDIT = 90;
 
 /**

src/grader/grader.js

@@ -1,35 +1,26 @@
 import { Expectation } from "../types.js";
-import {
-  GRADE_CHART,
-  LIKELIHOOD_INDICATOR_CHART,
-  SCORE_TABLE,
-  TEST_TOPIC_LINKS,
-} from "./charts.js";
+import { GRADE_CHART, SCORE_TABLE, TEST_TOPIC_LINKS } from "./charts.js";
 
 /**
- * @typedef {Object} GradeAndLikelihood
+ * @typedef {Object} GradeAndScore
  * @property {number} score
  * @property {string} grade
- * @property {string} likelihoodIndicator
  */
 
 /**
  *
  * @param {number} score - raw score based on all of the tests
- * @returns {GradeAndLikelihood} - normalized score, grade and likelihood_indicator
+ * @returns {GradeAndScore} - normalized score and grade
  */
-export function getGradeAndLikelihoodForScore(score) {
+export function getGradeForScore(score) {
   score = Math.max(score, 0);
 
   // If score>100, just use the grade for 100, otherwise round down to the nearest multiple of 5
   const grade = GRADE_CHART.get(Math.min(score - (score % 5), 100));
-  const likelihoodIndicator =
-    LIKELIHOOD_INDICATOR_CHART.get(grade.slice(0, 1)) ?? "UNKNOWN";
 
   return {
     score,
     grade,
-    likelihoodIndicator,
   };
 }
 

src/scanner/index.js

@@ -1,6 +1,6 @@
 import { MINIMUM_SCORE_FOR_EXTRA_CREDIT } from "../grader/charts.js";
 import {
-  getGradeAndLikelihoodForScore,
+  getGradeForScore,
   getScoreDescription,
   getScoreModifier,
 } from "../grader/grader.js";
@@ -72,7 +72,7 @@ export async function scan(hostname, options) {
       ? scoreWithExtraCredit
       : uncurvedScore;
 
-  const final = getGradeAndLikelihoodForScore(score);
+  const final = getGradeForScore(score);
 
   const tests = results.reduce((obj, result) => {
     const name = result.constructor.name;
@@ -85,7 +85,6 @@ export async function scan(hostname, options) {
       algorithmVersion: ALGORITHM_VERSION,
       grade: final.grade,
       error: null,
-      likelihoodIndicator: final.likelihoodIndicator,
       score: final.score,
       statusCode: statusCode,
       testsFailed: NUM_TESTS - testsPassed,

src/types.js

@@ -281,7 +281,6 @@ export class Policy {
  * @prop {number} algorithmVersion
  * @prop {string | null} error
  * @prop {string} grade
- * @prop {string} likelihoodIndicator
  * @prop {StringMap} responseHeaders
  * @prop {number} score
  * @prop {number} testsFailed
@@ -349,10 +348,8 @@ export class Policy {
  * @property {number} tests_quantity
  * @property {string} [grade]
  * @property {number} [score]
- * @property {string} [likelihood_indicator]
  * @property {string} [error]
  * @property {StringMap} response_headers
- * @property {boolean} hidden
  * @property {number} [status_code]
  */