cosa2stream: Rework to better optimize openshift/installer flow

When I was writing this originally I was focused on the goal
of converting cosa metadata to stream format, hence the name etc.

However upon trying to use this for openshift/installer I realized
something important: *after* we do the switch, the installer git
will be a "source of truth" for the pinned bootimages.

The expected user story here will be e.g. "update x86_64 to <version>".

So now we support this flow:
```
$ plume cosa2stream --distro rhcos --output data/data/rhcos.json x86_64=48.83.202102192019-0 s390x=47.83.202102090311-0
```

which hardcodes the URL we use now, and makes the arch+version mapping
explicit.  We also paper over the fact that currently for RHCOS
there are separate cosa builds per architecture; the non-x86_64 ones
are named e.g `rhcos-4.8-s390x` instead of just `rhcos-4.8`.

mantle/cmd/plume/cosa2stream.go

@@ -16,17 +16,25 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/coreos/stream-metadata-go/release"
 	"github.com/coreos/stream-metadata-go/stream"
 	"github.com/spf13/cobra"
 )
 
+const (
+	// This will hopefully migrate to mirror.openshift.com, see https://github.com/openshift/os/issues/477
+	rhcosCosaEndpoint = "https://releases-art-rhcos.svc.ci.openshift.org/art/storage/releases"
+)
+
 var (
 	cmdCosaBuildToStream = &cobra.Command{
 		Use:   "cosa2stream [options]",
@@ -38,29 +46,85 @@ var (
 
 	streamBaseURL string
 	streamName    string
+	distro        string
+	target        string
 )
 
 func init() {
 	cmdCosaBuildToStream.Flags().StringVar(&streamBaseURL, "url", "", "Base URL for build")
 	cmdCosaBuildToStream.Flags().StringVar(&streamName, "name", "", "Stream name")
-	cmdCosaBuildToStream.MarkFlagRequired("name")
+	cmdCosaBuildToStream.Flags().StringVar(&distro, "distro", "", "Distribution (fcos, rhcos)")
+	cmdCosaBuildToStream.Flags().StringVar(&target, "target", "", "Modify this file in place (default: no source, print to stdout)")
 	root.AddCommand(cmdCosaBuildToStream)
 }
 
 func runCosaBuildToStream(cmd *cobra.Command, args []string) error {
-	childArgs := []string{"generate-release-meta", "--stream-name=" + streamName}
+	var outStream stream.Stream
+
+	if target != "" {
+		buf, err := ioutil.ReadFile(target)
+		if err != nil {
+			return err
+		}
+		err = json.Unmarshal(buf, &outStream)
+		if err != nil {
+			return err
+		}
+	} else {
+		if streamName == "" {
+			return fmt.Errorf("--name must be provided (if no input file)")
+		}
+		outStream = stream.Stream{
+			Stream:        streamName,
+			Architectures: make(map[string]stream.Arch),
+		}
+	}
+	streamArches := outStream.Architectures
+
+	outStream.Metadata = stream.Metadata{LastModified: time.Now().UTC().Format(time.RFC3339)}
+
+	childArgs := []string{"generate-release-meta"}
+	if distro != "" {
+		childArgs = append(childArgs, "--distro="+distro)
+	}
 	if streamBaseURL != "" {
 		childArgs = append(childArgs, "--stream-baseurl="+streamBaseURL)
 	}
 
-	streamArches := make(map[string]stream.Arch)
 	for _, arg := range args {
 		releaseTmpf, err := ioutil.TempFile("", "release")
 		if err != nil {
 			return err
 		}
+		var archStreamName = streamName
+		if !strings.HasPrefix(arg, "https://") {
+			if distro != "rhcos" {
+				return errors.New("Arguments must be https:// URLs (or with --distro rhcos, ARCH=VERSION)")
+			}
+			parts := strings.SplitN(arg, "=", 2)
+			if len(parts) < 2 {
+				return fmt.Errorf("Expecting ARCH=VERSION, found: %s", arg)
+			}
+			arch := parts[0]
+			ver := parts[1]
+			// Convert e.g. 48.82.<timestamp> to rhcos-4.8
+			archStreamName = fmt.Sprintf("rhcos-%s.%s", ver[0:1], ver[1:2])
+			if arch != "x86_64" {
+				archStreamName += "-" + arch
+			}
+			endpoint := rhcosCosaEndpoint
+			if streamBaseURL != "" {
+				endpoint = streamBaseURL
+			}
+			base := fmt.Sprintf("%s/%s", endpoint, archStreamName)
+			u := fmt.Sprintf("%s/%s/%s/meta.json", base, ver, arch)
+			arg = u
+			childArgs = append(childArgs, "--stream-baseurl="+endpoint)
+		}
 		cosaArgs := append([]string{}, childArgs...)
-		cosaArgs = append(cosaArgs, []string{fmt.Sprintf("--url=" + arg), "--output=" + releaseTmpf.Name()}...)
+		cosaArgs = append(cosaArgs, "--url="+arg)
+		cosaArgs = append(cosaArgs, "--stream-name="+archStreamName)
+		cosaArgs = append(cosaArgs, "--output="+releaseTmpf.Name())
 		c := exec.Command("cosa", cosaArgs...)
 		c.Stderr = os.Stderr
 		if err := c.Run(); err != nil {
@@ -78,21 +142,27 @@ func runCosaBuildToStream(cmd *cobra.Command, args []string) error {
 		relarches := rel.ToStreamArchitectures()
 		for arch, relarchdata := range relarches {
 			if _, ok := streamArches[arch]; ok {
-				return fmt.Errorf("Duplicate architecture %s", arch)
+				if target == "" {
+					return fmt.Errorf("Duplicate architecture %s", arch)
+				}
 			}
 			streamArches[arch] = relarchdata
 		}
 	}
 
-	// Generate output stream from release
-	outStream := stream.Stream{
-		Stream:        streamName,
-		Metadata:      stream.Metadata{LastModified: time.Now().UTC().Format(time.RFC3339)},
-		Architectures: streamArches,
-	}
-
 	// Serialize to JSON
-	encoder := json.NewEncoder(os.Stdout)
+	var targetWriter io.Writer
+	if target != "" {
+		var err error
+		targetWriter, err = os.Create(target)
+		if err != nil {
+			return err
+		}
+	} else {
+		targetWriter = os.Stdout
+	}
+	encoder := json.NewEncoder(targetWriter)
+	encoder.SetIndent("", "  ")
 	if err := encoder.Encode(&outStream); err != nil {
 		return fmt.Errorf("Error while encoding: %v", err)
 	}

src/cmd-generate-release-meta

@@ -27,7 +27,11 @@ def ensure_dup(inp, out, inp_key, out_key):
 
 
 def url_builder(stream, version, arch, path):
-    return f"{args.stream_baseurl}/{stream}/builds/{version}/{arch}/{path}"
+    # This is a bug to be fixed with later work on https://github.com/openshift/os/issues/477
+    if args.distro == 'rhcos':
+        return f"{args.stream_baseurl}/{stream}/{version}/{arch}/{path}"
+    else:
+        return f"{args.stream_baseurl}/{stream}/builds/{version}/{arch}/{path}"
 
 
 def get_extension(path, modifier, arch):
@@ -37,8 +41,9 @@ def get_extension(path, modifier, arch):
 parser = ArgumentParser()
 parser.add_argument("--workdir", help="cosa workdir")
 parser.add_argument("--build-id", help="build id")
+parser.add_argument("--distro", help="Distro selects stream defaults such as baseurl and format", choices=['fcos', 'rhcos'])
 parser.add_argument("--stream-name", help="Override the stream ID (default is derived from coreos-assembler)")
-parser.add_argument("--stream-baseurl", help="Prefix URL for stream content", default=FCOS_STREAM_ENDPOINT)
+parser.add_argument("--stream-baseurl", help="Override prefix URL for stream content", default=FCOS_STREAM_ENDPOINT)
 parser.add_argument("--output", help="Output to file; default is build directory")
 parser.add_argument("--url", help="URL to a coreos-assembler meta.json", default=[], action='append')
 args = parser.parse_args()
@@ -78,7 +83,9 @@ if len(args.url) == 0:
 else:
     for url in args.url:
         print(f"Downloading {url}...")
-        parsed_builds.append(requests.get(url).json())
+        r = requests.get(url)
+        r.raise_for_status()
+        parsed_builds.append(r.json())
 
 # If any existing data, inherit it (if it's non-empty)
 if os.path.exists(args.output) and os.stat(args.output).st_size > 0: