Handling upload errors in logging

.changeset/mighty-goats-sell.md

@@ -0,0 +1,7 @@
+---
+"@siteimprove/alfa-test-utils": minor
+---
+
+**Breaking:** `SUP.upload` now returns a `Result<string, Array<string>>`; `Logging.from*` now accept a `Result<string, Array<string>>`.
+
+This allows for more than one error to be reported at once.

.changeset/strange-numbers-poke.md

@@ -0,0 +1,5 @@
+---
+"@siteimprove/alfa-test-utils": minor
+---
+
+**Added:** `Logging` now includes error messages about problems that happens during deserialization or upload of results.

.changeset/tidy-buses-approve.md

@@ -0,0 +1,5 @@
+---
+"@siteimprove/alfa-test-utils": minor
+---
+
+**Added:** `Logging` now accepts a severity (default to `"log"`) and `Logging#print` respects it.

docs/review/api/alfa-test-utils.api.md

@@ -10,6 +10,7 @@ import { Array as Array_2 } from '@siteimprove/alfa-array';
 import type { AxiosRequestConfig } from 'axios';
 import { Equatable } from '@siteimprove/alfa-equatable';
 import { Flattened } from '@siteimprove/alfa-rules';
+import { Iterable as Iterable_2 } from '@siteimprove/alfa-iterable';
 import * as json from '@siteimprove/alfa-json';
 import { Map as Map_2 } from '@siteimprove/alfa-map';
 import { Node } from '@siteimprove/alfa-dom';
@@ -104,16 +105,18 @@ export interface CommitInformation {
 }
 
 // @public
-export class Logging implements Equatable, json.Serializable<Logging.JSON> {
-    protected constructor(title: string, logs: Sequence<Logging>);
+export class Logging<S extends Logging.Severity = Logging.Severity> implements Equatable, json.Serializable<Logging.JSON> {
+    protected constructor(title: string, logs: Sequence<Logging>, severity: S);
     // (undocumented)
     equals(value: Logging): boolean;
     // (undocumented)
     equals(value: unknown): value is this;
     // (undocumented)
-    get logs(): Iterable<Logging>;
+    get logs(): Iterable_2<Logging>;
     // (undocumented)
-    static of(title: string, logs?: Iterable<Logging>): Logging;
+    static of(title: string, logs?: Iterable_2<Logging>): Logging<"log">;
+    // (undocumented)
+    static of<S extends Logging.Severity = "log">(title: string, severity: S, logs?: Iterable_2<Logging>): Logging<S>;
     // (undocumented)
     print(): void;
     // (undocumented)
@@ -130,11 +133,13 @@ export namespace Logging {
         Title = "Untitled";
     }
     // @internal (undocumented)
+    export function errorTitle(n: number): string;
+    // @internal (undocumented)
     export function fromAggregate(aggregate: Array_2<[string, {
         failed: number;
-    }]>, pageTitle?: string, pageReportUrl?: Result<string, string> | string): Logging;
+    }]>, pageTitle?: string, pageReportUrl?: Result<string, Array_2<string>> | string): Logging;
     // (undocumented)
-    export function fromAudit(audit: Audit | Audit.JSON, pageReportUrl?: Result<string, string> | string, options?: Options): Logging;
+    export function fromAudit(audit: Audit | Audit.JSON, pageReportUrl?: Result<string, Array_2<string>> | string, options?: Options): Logging;
     // (undocumented)
     export function isLogging(value: unknown): value is Logging;
     // @internal (undocumented)
@@ -146,12 +151,16 @@ export namespace Logging {
         // (undocumented)
         logs: Sequence.JSON<JSON>;
         // (undocumented)
+        severity: Severity;
+        // (undocumented)
         title: string;
     }
     // (undocumented)
     export interface Options {
         pageTitle?: string | ((page: Page) => string);
     }
+    // (undocumented)
+    export type Severity = "info" | "log" | "warn" | "error";
 }
 
 // @public
@@ -204,14 +213,18 @@ export namespace SIP {
         Title = "";
         const // (undocumented)
         Name: undefined;
+        // (undocumented)
+        export function missingOptions(missing: Array_2<string>): string;
+        const // (undocumented)
+        badCredentials = "Unauthorized request: the request was made with invalid credentials, verify your username and API key";
     }
     // @internal
     export namespace Metadata {
         export function axiosConfig(audit: Audit | Audit.JSON, options: Options, override: {
             url?: string;
             timestamp?: string;
             httpsAgent?: Agent;
-        }): Promise<AxiosRequestConfig>;
+        }): Result<AxiosRequestConfig, string>;
         // Warning: (ae-forgotten-export) The symbol "CamelCase" needs to be exported by the entry point index.d.ts
         //
         // (undocumented)
@@ -236,7 +249,7 @@ export namespace SIP {
             TestName?: string;
             Version: `${number}.${number}.${number}`;
         }
-        export function payload(audit: Audit | Audit.JSON, options: Partial<Options>, timestamp: string): Promise<Payload>;
+        export function payload(audit: Audit | Audit.JSON, options: Partial<Options>, timestamp: string): Result<Payload, string>;
             {};
     }
     // (undocumented)
@@ -265,13 +278,13 @@ export namespace SIP {
         export function payload(Id: string, audit: Audit | Audit.JSON): Payload;
             {};
     }
-    export function upload(audit: Audit | Audit.JSON, options: Options): Promise<Result<string, string>>;
+    export function upload(audit: Audit | Audit.JSON, options: Options): Promise<Result<string, Array_2<string>>>;
     // @internal
     export function upload(audit: Audit | Audit.JSON, options: Options, override: {
         url?: string;
         timestamp?: string;
         httpsAgent?: Agent;
-    }): Promise<Result<string, string>>;
+    }): Promise<Result<string, Array_2<string>>>;
 }
 
 // (No @packageDocumentation comment for this package)

packages/alfa-test-utils/package.json

@@ -66,7 +66,6 @@
     "@siteimprove/alfa-selective": "^0.97.0",
     "@siteimprove/alfa-selector": "^0.97.0",
     "@siteimprove/alfa-sequence": "^0.97.0",
-    "@siteimprove/alfa-thunk": "^0.97.0",
     "@siteimprove/alfa-wcag": "^0.97.0",
     "@siteimprove/alfa-web": "^0.97.0",
     "axios": "^1.7.4",

packages/alfa-test-utils/src/report/logging.ts

@@ -1,11 +1,11 @@
 import { Array } from "@siteimprove/alfa-array";
 import { Element, Query } from "@siteimprove/alfa-dom";
 import { Equatable } from "@siteimprove/alfa-equatable";
-import { Result } from "@siteimprove/alfa-result";
-import { Sequence } from "@siteimprove/alfa-sequence";
+import { Iterable } from "@siteimprove/alfa-iterable";
 
 import * as json from "@siteimprove/alfa-json";
-import type { Thunk } from "@siteimprove/alfa-thunk";
+import { Err, Ok, Result } from "@siteimprove/alfa-result";
+import { Sequence } from "@siteimprove/alfa-sequence";
 import { Page } from "@siteimprove/alfa-web";
 
 import chalk from "chalk";
@@ -18,17 +18,39 @@ import { getRuleTitle } from "./get-rule-title.js";
  *
  * @public
  */
-export class Logging implements Equatable, json.Serializable<Logging.JSON> {
-  public static of(title: string, logs?: Iterable<Logging>): Logging {
-    return new Logging(title, Sequence.from(logs ?? []));
+export class Logging<S extends Logging.Severity = Logging.Severity>
+  implements Equatable, json.Serializable<Logging.JSON>
+{
+  public static of(title: string, logs?: Iterable<Logging>): Logging<"log">;
+
+  public static of<S extends Logging.Severity = "log">(
+    title: string,
+    severity: S,
+    logs?: Iterable<Logging>
+  ): Logging<S>;
+
+  public static of<S extends Logging.Severity = "log">(
+    title: string,
+    severityOrLogs?: S | Iterable<Logging>,
+    logs?: Iterable<Logging>
+  ): Logging<S | "log"> {
+    const innerLogs: Iterable<Logging> =
+      typeof severityOrLogs === "string" ? logs ?? [] : severityOrLogs ?? [];
+
+    const severity =
+      typeof severityOrLogs === "string" ? severityOrLogs : "log";
+
+    return new Logging(title, Sequence.from(innerLogs), severity);
   }
 
   private readonly _title: string;
   private readonly _logs: Sequence<Logging>;
+  private readonly _severity: S;
 
-  protected constructor(title: string, logs: Sequence<Logging>) {
+  protected constructor(title: string, logs: Sequence<Logging>, severity: S) {
     this._title = title;
     this._logs = logs;
+    this._severity = severity;
   }
 
   public get title(): string {
@@ -40,9 +62,13 @@ export class Logging implements Equatable, json.Serializable<Logging.JSON> {
   }
 
   public print(): void {
-    console.group(this._title);
-    this._logs.forEach((log) => log.print());
-    console.groupEnd();
+    if (this._logs.isEmpty()) {
+      console[this._severity](this._title);
+    } else {
+      console.group(this._title);
+      this._logs.forEach((log) => log.print());
+      console.groupEnd();
+    }
   }
 
   public equals(value: Logging): boolean;
@@ -60,6 +86,7 @@ export class Logging implements Equatable, json.Serializable<Logging.JSON> {
   public toJSON(): Logging.JSON {
     return {
       title: this._title,
+      severity: this._severity,
       logs: this._logs.toJSON(),
     };
   }
@@ -72,9 +99,15 @@ export namespace Logging {
   export interface JSON {
     [name: string]: json.JSON;
     title: string;
+    severity: Severity;
     logs: Sequence.JSON<JSON>;
   }
 
+  /**
+   * {@link https://console.spec.whatwg.org/#loglevel-severity}
+   */
+  export type Severity = "info" | "log" | "warn" | "error";
+
   /** @internal */
   export namespace Defaults {
     export const Title = "Untitled";
@@ -99,22 +132,45 @@ export namespace Logging {
     );
   }
 
+  /**
+   * @internal
+   */
+  export function errorTitle(n: number): string {
+    return (
+      "The following problem" +
+      (n === 1 ? " was " : "s were ") +
+      "encountered while uploading results to the Siteimprove Intelligence Platform:"
+    );
+  }
+
   /**
    * @internal
    */
   export function fromAggregate(
     aggregate: Array<[string, { failed: number }]>,
     pageTitle?: string,
-    pageReportUrl?: Result<string, string> | string
+    pageReportUrl?: Result<string, Array<string>> | string
   ): Logging {
     return Logging.of("Siteimprove found accessibility issues:", [
+      // Show the page title
       Logging.of(chalk.bold(`Page - ${pageTitle ?? Defaults.Title}`)),
+
+      // Show any error during upload: missing or invalid credentials, etc.
+      ...(Err.isErr<Array<string>>(pageReportUrl)
+        ? [
+            Logging.of(
+              errorTitle(pageReportUrl.getErr().length),
+              pageReportUrl.getErr().map((error) => Logging.of(error, "warn"))
+            ),
+          ]
+        : []),
+
       // "This page contains X issues: URL" (if URL)
       // "This page contains X issues." (otherwise)
       Logging.of(
         `This page contains ${aggregate.length} issues${
           Result.isOk(pageReportUrl)
-            ? ": " + chalk.underline(pageReportUrl.getUnsafe())
+            ? ": " + chalk.underline(pageReportUrl.get())
             : "."
         }`,
         aggregate.map(([ruleId, { failed }], index) =>
@@ -135,38 +191,48 @@ export namespace Logging {
 
   export function fromAudit(
     audit: Audit | Audit.JSON,
-    pageReportUrl?: Result<string, string> | string,
+    pageReportUrl?: Result<string, Array<string>> | string,
     options?: Options
   ): Logging {
-    const page: Thunk<Page> = () =>
-      Page.isPage(audit.page)
-        ? audit.page
-        : Page.from(audit.page).getUnsafe("Could not deserialize the page");
-    const title =
-      options?.pageTitle ??
-      Query.getElementDescendants(page().document)
-        .filter(Element.isElement)
-        .find(Element.hasName("title"))
-        .map((title) => title.textContent())
-        .getOr(Defaults.Title);
-    const pageTitle =
-      typeof title === "string"
-        ? title
-        : title !== undefined
-        ? title(page())
-        : title;
-
-    const filteredAggregates = Array.sortWith(
-      (Audit.isAudit(audit)
-        ? audit.resultAggregates.toArray()
-        : audit.resultAggregates
-      ).filter(([_, { failed }]) => failed > 0),
-      ([uria], [urib]) => uria.localeCompare(urib)
-    ).map(([url, aggregate]): [string, { failed: number }] => [
-      url.split("/").pop() ?? "",
-      aggregate,
-    ]);
-    return fromAggregate(filteredAggregates, pageTitle, pageReportUrl);
+    // Retrieve or deserialize the page
+    // We may waste a bit of time deserializing a page we won't need (if URL
+    // and title are provided), but this streamlines error handling.
+    const logs = (
+      Page.isPage(audit.page) ? Ok.of(audit.page) : Page.from(audit.page)
+    ).map((page) => {
+      const title =
+        options?.pageTitle ??
+        Query.getElementDescendants(page.document)
+          .filter(Element.isElement)
+          .find(Element.hasName("title"))
+          .map((title) => title.textContent())
+          .getOr(Defaults.Title);
+      const pageTitle =
+        typeof title === "string"
+          ? title
+          : title !== undefined
+          ? title(page)
+          : title;
+
+      const filteredAggregates = Array.sortWith(
+        (Audit.isAudit(audit)
+          ? audit.resultAggregates.toArray()
+          : audit.resultAggregates
+        ).filter(([_, { failed }]) => failed > 0),
+        ([uria], [urib]) => uria.localeCompare(urib)
+      ).map(([url, aggregate]): [string, { failed: number }] => [
+        url.split("/").pop() ?? "",
+        aggregate,
+      ]);
+      return fromAggregate(filteredAggregates, pageTitle, pageReportUrl);
+    });
+
+    return logs.getOrElse(() =>
+      Logging.of(
+        `Could not deserialize the page: ${logs.getErrUnsafe()}`,
+        "error"
+      )
+    );
   }
 
   /**