[DRAFT] Add documentation generation

go.mod

@@ -9,6 +9,7 @@ require (
 	github.com/evanw/esbuild v0.18.11
 	github.com/fsouza/go-dockerclient v1.9.8
 	github.com/gorilla/mux v1.8.0
+	github.com/lithammer/dedent v1.1.0
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	golang.org/x/net v0.17.0
 	gopkg.in/inconshreveable/log15.v2 v2.0.0-20200109203555-b30bc20e4fd1

go.sum

@@ -226,6 +226,8 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0
 github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
 github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c=
+github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
+github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=

hive.go

@@ -17,6 +17,10 @@ import (
 )
 
 func main() {
+	cwd, err := os.Getwd()
+	if err != nil {
+		fatal(err)
+	}
 	var (
 		testResultsRoot       = flag.String("results-root", "workspace/logs", "Target `directory` for results files and logs.")
 		loglevelFlag          = flag.Int("loglevel", 3, "Log `level` for system events. Supports values 0-5.")
@@ -33,6 +37,8 @@ func main() {
 		simLogLevel           = flag.Int("sim.loglevel", 3, "Selects log `level` of client instances. Supports values 0-5.")
 		simDevMode            = flag.Bool("dev", false, "Only starts the simulator API endpoint (listening at 127.0.0.1:3000 by default) without starting any simulators.")
 		simDevModeAPIEndpoint = flag.String("dev.addr", "127.0.0.1:3000", "Endpoint that the simulator API listens on")
+		simDocsMode           = flag.Bool("docs", false, "Starts the simulator in docs mode, where it will not execute any tests, but will instead generate documentation for the tests.")
+		simDocsOutput         = flag.String("docs.output", cwd, "Base output directory for generated documentation. Defaults to cwd.")
 		useCredHelper         = flag.Bool("docker.cred-helper", false, "configure docker authentication using locally-configured credential helper")
 
 		clientsFile = flag.String("client-file", "", `YAML `+"`file`"+` containing client configurations.`)
@@ -107,13 +113,15 @@ func main() {
 
 	// Run.
 	env := libhive.SimEnv{
-		LogDir:             *testResultsRoot,
-		SimLogLevel:        *simLogLevel,
-		SimTestPattern:     *simTestPattern,
-		SimParallelism:     *simParallelism,
-		SimRandomSeed:      *simRandomSeed,
-		SimDurationLimit:   *simTimeLimit,
-		ClientStartTimeout: *clientTimeout,
+		LogDir:               *testResultsRoot,
+		SimLogLevel:          *simLogLevel,
+		SimTestPattern:       *simTestPattern,
+		SimParallelism:       *simParallelism,
+		SimRandomSeed:        *simRandomSeed,
+		SimDurationLimit:     *simTimeLimit,
+		SimDocsMode:          *simDocsMode,
+		SimDocsOutputBaseDir: *simDocsOutput,
+		ClientStartTimeout:   *clientTimeout,
 	}
 	runner := libhive.NewRunner(inv, builder, cb)
 

hivesim/hive.go

@@ -21,9 +21,10 @@ import (
 
 // Simulation wraps the simulation HTTP API provided by hive.
 type Simulation struct {
-	url      string
-	m        testMatcher
-	ll int
+	url  string
+	m    testMatcher
+	docs bool
+	ll   int
 }
 
 // New looks up the hive host URI using the HIVE_SIMULATOR environment variable
@@ -47,6 +48,8 @@ func New() *Simulation {
 	if ll := os.Getenv("HIVE_LOGLEVEL"); ll != "" {
 		sim.ll, _ = strconv.Atoi(ll)
 	}
+	cc := os.Getenv("HIVE_DOCS_MODE")
+	sim.docs = (cc == "true")
 	return sim
 }
 
@@ -88,10 +91,10 @@ func (sim *Simulation) EndTest(testSuite SuiteID, test TestID, testResult TestRe
 }
 
 // StartSuite signals the start of a test suite.
-func (sim *Simulation) StartSuite(name, description, simlog string) (SuiteID, error) {
+func (sim *Simulation) StartSuite(suite *simapi.TestRequest, simlog string) (SuiteID, error) {
 	var (
 		url  = fmt.Sprintf("%s/testsuite", sim.url)
-		req  = &simapi.TestRequest{Name: name, Description: description}
+		req  = suite
 		resp SuiteID
 	)
 	err := post(url, req, &resp)
@@ -105,10 +108,10 @@ func (sim *Simulation) EndSuite(testSuite SuiteID) error {
 }
 
 // StartTest starts a new test case, returning the testcase id as a context identifier.
-func (sim *Simulation) StartTest(testSuite SuiteID, name string, description string) (TestID, error) {
+func (sim *Simulation) StartTest(testSuite SuiteID, test *simapi.TestRequest) (TestID, error) {
 	var (
 		url  = fmt.Sprintf("%s/testsuite/%d/test", sim.url, testSuite)
-		req  = &simapi.TestRequest{Name: name, Description: description}
+		req  = test
 		resp TestID
 	)
 	err := post(url, req, &resp)

hivesim/hive_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/hive/internal/fakes"
 	"github.com/ethereum/hive/internal/libhive"
+	"github.com/ethereum/hive/internal/simapi"
 )
 
 // This test checks that the API returns configured client names correctly.
@@ -73,11 +74,11 @@ func TestEnodeReplaceIP(t *testing.T) {
 
 	// Start the client.
 	sim := NewAt(srv.URL)
-	suiteID, err := sim.StartSuite("suite", "", "")
+	suiteID, err := sim.StartSuite(&simapi.TestRequest{Name: "suite"}, "")
 	if err != nil {
 		t.Fatal("can't start suite:", err)
 	}
-	testID, err := sim.StartTest(suiteID, "test", "")
+	testID, err := sim.StartTest(suiteID, &simapi.TestRequest{Name: "test"})
 	if err != nil {
 		t.Fatal("can't start test:", err)
 	}
@@ -125,11 +126,11 @@ func TestStartClientStartOptions(t *testing.T) {
 
 	// Start the suite and test.
 	sim := NewAt(srv.URL)
-	suiteID, err := sim.StartSuite("suite", "", "")
+	suiteID, err := sim.StartSuite(&simapi.TestRequest{Name: "suite"}, "")
 	if err != nil {
 		t.Fatal("can't start suite:", err)
 	}
-	testID, err := sim.StartTest(suiteID, "test", "")
+	testID, err := sim.StartTest(suiteID, &simapi.TestRequest{Name: "test"})
 	if err != nil {
 		t.Fatal("can't start test:", err)
 	}
@@ -262,11 +263,11 @@ func TestRunProgram(t *testing.T) {
 	defer tm.Terminate()
 
 	sim := NewAt(srv.URL)
-	suiteID, err := sim.StartSuite("suite", "", "")
+	suiteID, err := sim.StartSuite(&simapi.TestRequest{Name: "suite"}, "")
 	if err != nil {
 		t.Fatal("can't start suite:", err)
 	}
-	testID, err := sim.StartTest(suiteID, "test", "")
+	testID, err := sim.StartTest(suiteID, &simapi.TestRequest{Name: "test"})
 	if err != nil {
 		t.Fatal("can't start test:", err)
 	}
@@ -309,11 +310,11 @@ func TestStartClientErrors(t *testing.T) {
 	defer tm.Terminate()
 
 	sim := NewAt(srv.URL)
-	suiteID, err := sim.StartSuite("suite", "", "")
+	suiteID, err := sim.StartSuite(&simapi.TestRequest{Name: "suite"}, "")
 	if err != nil {
 		t.Fatal("can't start suite:", err)
 	}
-	testID, err := sim.StartTest(suiteID, "test", "")
+	testID, err := sim.StartTest(suiteID, &simapi.TestRequest{Name: "test"})
 	if err != nil {
 		t.Fatal("can't start test:", err)
 	}
@@ -365,11 +366,11 @@ func TestStartClientInitialNetworks(t *testing.T) {
 	defer tm.Terminate()
 
 	sim := NewAt(srv.URL)
-	suiteID, err := sim.StartSuite("suite", "", "")
+	suiteID, err := sim.StartSuite(&simapi.TestRequest{Name: "suite"}, "")
 	if err != nil {
 		t.Fatal("can't start suite:", err)
 	}
-	testID, err := sim.StartTest(suiteID, "test", "")
+	testID, err := sim.StartTest(suiteID, &simapi.TestRequest{Name: "test"})
 	if err != nil {
 		t.Fatal("can't start test:", err)
 	}

hivesim/testapi.go

@@ -9,15 +9,27 @@ import (
 	"sync"
 
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/hive/internal/simapi"
 )
 
 // Suite is the description of a test suite.
 type Suite struct {
 	Name        string
+	DisplayName string
+	Category    string
 	Description string
 	Tests       []AnyTest
 }
 
+func (s *Suite) request() *simapi.TestRequest {
+	return &simapi.TestRequest{
+		Name:        s.Name,
+		DisplayName: s.DisplayName,
+		Category:    s.Category,
+		Description: s.Description,
+	}
+}
+
 // Add adds a test to the suite.
 func (s *Suite) Add(test AnyTest) *Suite {
 	s.Tests = append(s.Tests, test)
@@ -56,7 +68,7 @@ func RunSuite(host *Simulation, suite Suite) error {
 		return nil
 	}
 
-	suiteID, err := host.StartSuite(suite.Name, suite.Description, "")
+	suiteID, err := host.StartSuite(suite.request(), "")
 	if err != nil {
 		return err
 	}
@@ -93,8 +105,10 @@ func MustRunSuite(host *Simulation, suite Suite) {
 type TestSpec struct {
 	// These fields are displayed in the UI. Be sure to add
 	// a meaningful description here.
-	Name        string
-	Description string
+	Name        string // Name is the unique identifier for the test [Mandatory]
+	DisplayName string // Display name for the test (Name will be used if unset) [Optional]
+	Description string // Description of the test (if empty, test won't appear in documentation) [Optional]
+	Category    string // Category of the test [Optional]
 
 	// If AlwaysRun is true, the test will run even if Name does not match the test
 	// pattern. This option is useful for tests that launch a client instance and
@@ -115,8 +129,10 @@ type TestSpec struct {
 type ClientTestSpec struct {
 	// These fields are displayed in the UI. Be sure to add
 	// a meaningful description here.
-	Name        string
-	Description string
+	Name        string // Name is the unique identifier for the test [Mandatory]
+	DisplayName string // Display name for the test (Name will be used if unset) [Optional]
+	Description string // Description of the test (if empty, test won't appear in documentation) [Optional]
+	Category    string // Category of the test [Optional]
 
 	// If AlwaysRun is true, the test will run even if Name does not match the test
 	// pattern. This option is useful for tests that launch a client instance and
@@ -208,11 +224,13 @@ func (t *T) StartClient(clientType string, option ...StartOption) *Client {
 // It waits for the subtest to complete.
 func (t *T) RunClient(clientType string, spec ClientTestSpec) {
 	test := testSpec{
-		suiteID:   t.SuiteID,
-		suite:     t.suite,
-		name:      clientTestName(spec.Name, clientType),
-		desc:      spec.Description,
-		alwaysRun: spec.AlwaysRun,
+		suiteID:     t.SuiteID,
+		suite:       t.suite,
+		name:        clientTestName(spec.Name, clientType),
+		displayName: spec.DisplayName,
+		category:    spec.Category,
+		desc:        spec.Description,
+		alwaysRun:   spec.AlwaysRun,
 	}
 	runTest(t.Sim, test, func(t *T) {
 		client := t.StartClient(clientType, spec.Parameters, WithStaticFiles(spec.Files))
@@ -298,11 +316,22 @@ func (t *T) FailNow() {
 }
 
 type testSpec struct {
-	suiteID   SuiteID
-	suite     *Suite
-	name      string
-	desc      string
-	alwaysRun bool
+	suiteID     SuiteID
+	suite       *Suite
+	name        string
+	displayName string
+	category    string
+	desc        string
+	alwaysRun   bool
+}
+
+func (spec testSpec) request() *simapi.TestRequest {
+	return &simapi.TestRequest{
+		Name:        spec.name,
+		DisplayName: spec.displayName,
+		Category:    spec.category,
+		Description: spec.desc,
+	}
 }
 
 func runTest(host *Simulation, test testSpec, runit func(t *T)) error {
@@ -319,7 +348,7 @@ func runTest(host *Simulation, test testSpec, runit func(t *T)) error {
 		SuiteID: test.suiteID,
 		suite:   test.suite,
 	}
-	testID, err := host.StartTest(test.suiteID, test.name, test.desc)
+	testID, err := host.StartTest(test.suiteID, test.request())
 	if err != nil {
 		return err
 	}
@@ -343,6 +372,10 @@ func runTest(host *Simulation, test testSpec, runit func(t *T)) error {
 			}
 			close(done)
 		}()
+		if host.docs && !test.alwaysRun {
+			// Don't run the test if we're just generating docs.
+			return
+		}
 		runit(t)
 	}()
 	<-done
@@ -361,11 +394,13 @@ func (spec ClientTestSpec) runTest(host *Simulation, suiteID SuiteID, suite *Sui
 			continue
 		}
 		test := testSpec{
-			suiteID:   suiteID,
-			suite:     suite,
-			name:      clientTestName(spec.Name, clientDef.Name),
-			desc:      spec.Description,
-			alwaysRun: spec.AlwaysRun,
+			suiteID:     suiteID,
+			suite:       suite,
+			name:        clientTestName(spec.Name, clientDef.Name),
+			displayName: spec.DisplayName,
+			category:    spec.Category,
+			desc:        spec.Description,
+			alwaysRun:   spec.AlwaysRun,
 		}
 		err := runTest(host, test, func(t *T) {
 			client := t.StartClient(clientDef.Name, spec.Parameters, WithStaticFiles(spec.Files))
@@ -391,11 +426,13 @@ func clientTestName(name, clientType string) string {
 
 func (spec TestSpec) runTest(host *Simulation, suiteID SuiteID, suite *Suite) error {
 	test := testSpec{
-		suiteID:   suiteID,
-		suite:     suite,
-		name:      spec.Name,
-		desc:      spec.Description,
-		alwaysRun: spec.AlwaysRun,
+		suiteID:     suiteID,
+		suite:       suite,
+		name:        spec.Name,
+		displayName: spec.DisplayName,
+		category:    spec.Category,
+		desc:        spec.Description,
+		alwaysRun:   spec.AlwaysRun,
 	}
 	return runTest(host, test, spec.Run)
 }

internal/libhive/api.go

@@ -75,7 +75,7 @@ func (api *simAPI) startSuite(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	suiteID, err := api.tm.StartTestSuite(suite.Name, suite.Description)
+	suiteID, err := api.tm.StartTestSuite(&suite)
 	if err != nil {
 		log15.Error("API: StartTestSuite failed", "error", err)
 		serveError(w, err, http.StatusInternalServerError)
@@ -117,7 +117,7 @@ func (api *simAPI) startTest(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	testID, err := api.tm.StartTest(suiteID, test.Name, test.Description)
+	testID, err := api.tm.StartTest(suiteID, &test)
 	if err != nil {
 		err := fmt.Errorf("can't start test case: %s", err.Error())
 		serveError(w, err, http.StatusInternalServerError)

internal/libhive/data.go

@@ -24,6 +24,8 @@ func (tsID TestID) String() string {
 type TestSuite struct {
 	ID             TestSuiteID          `json:"id"`
 	Name           string               `json:"name"`
+	DisplayName    string               `json:"displayName"`
+	Category       string               `json:"category"`
 	Description    string               `json:"description"`
 	ClientVersions map[string]string    `json:"clientVersions"`
 	TestCases      map[TestID]*TestCase `json:"testCases"`
@@ -38,6 +40,8 @@ type TestSuite struct {
 // TestCase represents a single test case in a test suite.
 type TestCase struct {
 	Name          string                 `json:"name"`        // Test case short name.
+	DisplayName   string                 `json:"displayName"` // Test case display name.
+	Category      string                 `json:"category"`    // Test case category.
 	Description   string                 `json:"description"` // Test case long description in MD.
 	Start         time.Time              `json:"start"`
 	End           time.Time              `json:"end"`