Document the expected Sigstore CLI interface

.github/workflows/selftest.yml → .github/workflows/conformance.yml

@@ -20,4 +20,4 @@ jobs:
         uses: ./
         id: sigstore-conformance
         with:
-          entrypoint: sigstore
+          entrypoint: ${{ github.workspace }}/sigstore-python-conformance

Makefile

@@ -1,4 +1,5 @@
 ALL_PY_SRCS := action.py \
+	sigstore-python-conformance \
 	$(shell find test/ -name '*.py')
 
 .PHONY: all

README.md

@@ -10,7 +10,8 @@ sigstore-conformance
 
 ## Usage
 
-Simply add `trailofbits/sigstore-conformance` to one of your workflows.
+Simply create a new workflow file at `.github/workflows/conformance.yml` and add
+the `trailofbits/sigstore-conformance` action to it.
 
 ```yaml
 jobs:
@@ -35,6 +36,10 @@ In the example above, the workflow is installing [sigstore-python](https://githu
 and providing `sigstore` as the `entrypoint` since this is the command used to
 invoke the client.
 
+The workflow that uses this action **must** be at
+`.github/workflows/conformance.yml`. This is a current limitation of the test
+suite and is required to reliably verify signing certificates.
+
 The relevant job must have permission to request the OIDC token to authenticate
 with. This can be done by adding a `permission` setting within the job that
 invokes the `trailofbits/sigstore-conformance` action.

docs/cli_protocol.md

@@ -0,0 +1,55 @@
+Conformance CLI Protocol
+========================
+
+While all Sigstore clients share some core functionality, each of them have
+their own unique features and idiosyncracies that make their command lines
+incompatible with one another. `sigstore-conformance` aims to test this common
+feature set.
+
+As an example of incompatibilities between clients, [cosign's](https://github.com/sigstore/cosign)
+`sign` subcommand is designed to sign Docker containers rather than arbitrary
+files (this behaviour is accessed via cosign's `sign-blob` subcommand). This
+makes sense for cosign's use case, however it does diverge from other Sigstore
+clients such as [sigstore-python](https://github.com/sigstore/sigstore-python)
+and [sigstore-js](https://github.com/sigstore/sigstore-js).
+
+To resolve these differences, we've established a CLI protocol for the test
+suite to uniformly communicate with clients under test. Since the client's CLI
+is unlikely to conform to this protocol, it may be necessary to write a thin
+wrapper that converts the options described in this document to those that the
+client's native CLI accepts.
+
+## Subcommands
+
+This is the set of subcommands that the test CLI must support. Each subcommand
+has a provided syntax and list of descriptions for each argument.
+
+To simplify argument parsing, all arguments are required and will **always** be
+supplied by the conformance suite in the order that they are specified in the
+templates below.
+
+### Sign
+
+```console
+${ENTRYPOINT} sign --signature FILE --certificate FILE FILE
+```
+
+| Option | Description |
+| --- | --- |
+| `--signature FILE` | The path to write the signature to |
+| `--certificate FILE` | The path to write the signing certificate to |
+| `FILE` | The artifact to sign |
+
+### Verify
+
+```console
+${ENTRYPOINT} verify --signature FILE --certificate FILE --certificate-oidc-issuer URL FILE
+```
+
+| Option | Description |
+| --- | --- |
+| `--signature FILE` | The path to the signature to verify |
+| `--certificate FILE` | The path to the signing certificate to verify |
+| `--certificate-email EMAIL` | The The expected email in the signing certificate's SAN extension |
+| `--certificate-oidc-issuer URL` | The expected OIDC issuer for the signing certificate |
+| `FILE` | The path to the artifact to verify |

sigstore-python-conformance

@@ -0,0 +1,26 @@
+#!/usr/bin/env python3
+
+"""
+A wrapper to convert `sigstore-conformance` CLI protocol invocations to match `sigstore-python`.
+"""
+
+import subprocess
+import sys
+
+ARG_REPLACEMENTS = {
+    "--certificate-email": "--cert-email",
+    "--certificate-oidc-issuer": "--cert-oidc-issuer",
+}
+
+# Trim the script name.
+fixed_args = sys.argv[1:]
+
+# Replace incompatible flags.
+fixed_args = [
+    ARG_REPLACEMENTS[arg] if arg in ARG_REPLACEMENTS else arg for arg in fixed_args
+]
+
+# Prepend the `sigstore-python` executable name.
+fixed_args = ["sigstore"] + fixed_args
+
+subprocess.run(fixed_args, text=True, check=True)

test/client.py

@@ -1,6 +1,15 @@
 import os
 import subprocess
 
+CERTIFICATE_EMAIL = (
+    os.environ["GITHUB_SERVER_URL"]
+    + "/"
+    + os.environ["GITHUB_REPOSITORY"]
+    + "/.github/workflows/conformance.yml@"
+    + os.environ["GITHUB_REF"]
+)
+CERTIFICATE_OIDC_ISSUER = "https://token.actions.githubusercontent.com"
+
 
 class SigstoreClient:
     """
@@ -32,29 +41,40 @@ def run(self, *args) -> None:
         )
 
     def sign(
-        self, artifact: os.PathLike, certificate: os.PathLike, signature: os.PathLike
+        self, artifact: os.PathLike, signature: os.PathLike, certificate: os.PathLike
     ) -> None:
         """
         Sign an artifact with the Sigstore client.
 
         `artifact` is a path to the file to sign.
-        `certificate` is the path to write the signing certificate to.
         `signature` is the path to write the generated signature to.
+        `certificate` is the path to write the signing certificate to.
         """
         self.run(
-            "sign", "--certificate", certificate, "--signature", signature, artifact
+            "sign", "--signature", signature, "--certificate", certificate, artifact
         )
 
     def verify(
-        self, artifact: os.PathLike, certificate: os.PathLike, signature: os.PathLike
+        self, artifact: os.PathLike, signature: os.PathLike, certificate: os.PathLike
     ) -> None:
         """
         Verify an artifact with the Sigstore client.
 
         `artifact` is the path to the file to verify.
-        `certificate` is the path to the signing certificate to verify with.
         `signature` is the path to the signature to verify.
+        `certificate` is the path to the signing certificate to verify with.
         """
+        # The email and OIDC issuer cannot be specified by the test since they remain constant
+        # across the GitHub Actions job.
         self.run(
-            "verify", "--certificate", certificate, "--signature", signature, artifact
+            "verify",
+            "--signature",
+            signature,
+            "--certificate",
+            certificate,
+            "--certificate-email",
+            CERTIFICATE_EMAIL,
+            "--certificate-oidc-issuer",
+            CERTIFICATE_OIDC_ISSUER,
+            artifact,
         )

test/test_sign_verify.py

@@ -9,18 +9,18 @@ def test_sign_verify(client: SigstoreClient) -> None:
     client.
     """
     artifact_path = Path("artifact.txt")
-    certificate_path = Path("artifact.txt.crt")
     signature_path = Path("artifact.txt.sig")
+    certificate_path = Path("artifact.txt.crt")
 
     assert artifact_path.exists()
-    assert not certificate_path.exists()
     assert not signature_path.exists()
+    assert not certificate_path.exists()
 
     # Sign the artifact.
-    client.sign(artifact_path, certificate_path, signature_path)
+    client.sign(artifact_path, signature_path, certificate_path)
 
-    assert certificate_path.exists()
     assert signature_path.exists()
+    assert certificate_path.exists()
 
-    # Verify the artifact signature
-    client.verify(artifact_path, certificate_path, signature_path)
+    # Verify the artifact signature.
+    client.verify(artifact_path, signature_path, certificate_path)