Use and modify docker-mirror to mirror repositories to public ECR registries

.github/workflows/CD.yml

@@ -343,6 +343,7 @@ jobs:
           docker tag $IMAGE_NAME public.ecr.aws/$ECR_REPO:latest 
           docker push public.ecr.aws/$ECR_REPO:$TAG
           docker push public.ecr.aws/$ECR_REPO:latest
+          go run tools/release/adot-operator-images-mirror
 
       - name: Login Dockerhub
         uses: docker/login-action@v1

THIRD-PARTY-LICENSES.txt

@@ -849,4 +849,36 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice
 ---------------------------------------------------------
 
   This Source Code Form is "Incompatible With Secondary Licenses", as
-  defined by the Mozilla Public License, v. 2.0.
+  defined by the Mozilla Public License, v. 2.0.
+
+------
+
+BSD 3-Clause License
+
+Copyright (c) 2017, SeatGeek
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+* Redistributions of source code must retain the above copyright notice, this
+  list of conditions and the following disclaimer.
+
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+
+* Neither the name of the copyright holder nor the names of its
+  contributors may be used to endorse or promote products derived from
+  this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

go.mod

@@ -3,7 +3,11 @@ module github.com/aws-observability/aws-otel-collector
 go 1.14
 
 require (
+	github.com/aws/aws-sdk-go-v2/config v1.5.0
+	github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.4.1
+	github.com/cenkalti/backoff v2.2.1+incompatible
 	github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b
+	github.com/fsouza/go-dockerclient v1.7.3
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter v0.30.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.30.0
 	github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.30.0
@@ -28,6 +32,7 @@ require (
 	go.uber.org/zap v1.18.1
 	golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0
+	gopkg.in/yaml.v2 v2.4.0
 )
 
 replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.30.0

tools/release/adot-operator-images-mirror/config.yaml

@@ -0,0 +1,14 @@
+## The source repositories and target repositories are one-to-one correspondence.
+sourceRepos:
+  - registry: opentelemetry
+    name: opentelemetry-operator
+    host: quay.io
+  - registry: kubebuilder
+    name: kube-rbac-proxy
+    host: gcr.io
+
+targetRepos:
+  - registry: public.ecr.aws/aws-observability
+    name: adot-operator
+  - registry: public.ecr.aws/aws-observability
+    name: mirror-kube-rbac-proxy

tools/release/adot-operator-images-mirror/docker_auth.go

@@ -0,0 +1,22 @@
+package main
+
+import (
+	"fmt"
+	"log"
+
+	docker "github.com/fsouza/go-dockerclient"
+)
+
+func getDockerCredentials(registry string) (*docker.AuthConfiguration, error) {
+	authOptions, err := docker.NewAuthConfigurationsFromDockerCfg()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	creds, ok := authOptions.Configs[registry]
+	if !ok {
+		return nil, fmt.Errorf("no auth found for %s", registry)
+	}
+
+	return &creds, nil
+}

tools/release/adot-operator-images-mirror/ecr_manager.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/aws/aws-sdk-go-v2/service/ecrpublic"
+	"github.com/cenkalti/backoff"
+)
+
+const (
+	operatorRepo  = "adot-operator"
+	rbacProxyRepo = "mirror-kube-rbac-proxy"
+)
+
+type ecrManager struct {
+	client       *ecrpublic.Client
+	repositories map[string]bool
+}
+
+func (e *ecrManager) exists(name string) bool {
+	if _, ok := e.repositories[name]; ok {
+		return true
+	}
+
+	return false
+}
+
+func (e *ecrManager) ensure(name string) error {
+	if e.exists(name) {
+		return nil
+	}
+
+	return e.create(name)
+}
+
+func (e *ecrManager) create(name string) error {
+	if name != operatorRepo && name != rbacProxyRepo {
+		return fmt.Errorf("wrong repository name: %s", name)
+	}
+
+	_, err := e.client.CreateRepository(context.TODO(), &ecrpublic.CreateRepositoryInput{
+		RepositoryName: &name,
+	})
+	if err != nil {
+		return err
+	}
+
+	e.repositories[name] = true
+	return nil
+}
+
+func (e *ecrManager) buildCache(nextToken *string) error {
+	if nextToken == nil {
+		log.Println("Loading the list of ECR repositories")
+	}
+
+	resp, err := e.client.DescribeRepositories(context.TODO(), &ecrpublic.DescribeRepositoriesInput{
+		NextToken: nextToken,
+	})
+	if err != nil {
+		return err
+	}
+
+	if e.repositories == nil {
+		e.repositories = make(map[string]bool)
+	}
+
+	for _, repo := range resp.Repositories {
+		e.repositories[*repo.RepositoryName] = true
+	}
+
+	// keep paging as long as there is a token for the next page
+	if resp.NextToken != nil {
+		err := e.buildCache(resp.NextToken)
+		if err != nil {
+			return err
+		}
+	}
+
+	// no next token means we hit the last page
+	if nextToken == nil {
+		log.Println("Done loading ECR repositories")
+	}
+
+	return nil
+}
+
+func (e *ecrManager) buildCacheBackoff() backoff.Operation {
+	return func() error {
+		return e.buildCache(nil)
+	}
+}

tools/release/adot-operator-images-mirror/main.go

@@ -0,0 +1,124 @@
+package main
+
+import (
+	"context"
+	"io/ioutil"
+	"log"
+	"runtime"
+	"sync"
+	"time"
+
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/service/ecrpublic"
+	"github.com/cenkalti/backoff"
+	docker "github.com/fsouza/go-dockerclient"
+	"gopkg.in/yaml.v2"
+)
+
+const ecrPublicRegion = "us-east-1"
+
+var (
+	config          Config
+	backoffSettings *backoff.ExponentialBackOff
+)
+
+type Config struct {
+	SourceRepos []Repository `yaml:"sourceRepos"`
+	TargetRepos []Repository `yaml:"targetRepos"`
+}
+
+type Repository struct {
+	Registry string `yaml:"registry"`
+	Name     string `yaml:"name"`
+	Host     string `yaml:"host"`
+}
+
+// Create the Docker client which will be used to pull, retag and push images.
+func createDockerClient() (*docker.Client, error) {
+	client, err := docker.NewClientFromEnv()
+	return client, err
+}
+
+// worker is the mirror working function.
+func worker(wg *sync.WaitGroup, workerCh chan []Repository, dc *DockerClient, ecrm *ecrManager) {
+	for repos := range workerCh {
+		log.Printf("Starting to mirror repo %s/%s/%s to repo %s/%s", repos[0].Host, repos[0].Registry, repos[0].Name, repos[1].Registry, repos[1].Name)
+
+		m := mirror{
+			dockerClient: dc,
+			ecrManager:   ecrm,
+		}
+		if err := m.setup(repos); err != nil {
+			log.Printf("Failed to setup mirror for repo %s/%s/%s: %v", repos[0].Host, repos[0].Registry, repos[0].Name, err)
+			wg.Done()
+			continue
+		}
+
+		m.work()
+		log.Printf("From repo %s/%s/%s to repo %s/%s mirror completed", repos[0].Host, repos[0].Registry, repos[0].Name, repos[1].Registry, repos[1].Name)
+		wg.Done()
+	}
+}
+
+func main() {
+	content, err := ioutil.ReadFile("tools/release/adot-operator-images-mirror/config.yaml")
+	if err != nil {
+		log.Fatalf("Could not read config file: %v", err)
+	}
+
+	if err = yaml.Unmarshal(content, &config); err != nil {
+		log.Fatalf("Could not parse config file: %v", err)
+	}
+
+	log.Println("Creating Docker client")
+	var client DockerClient
+	client, err = createDockerClient()
+	if err != nil {
+		log.Fatalf("Could not create Docker client: %v", err)
+	}
+
+	info, err := client.Info()
+	if err != nil {
+		log.Fatalf("Could not get Docker info: %v", err)
+	}
+	log.Printf("Connected to Docker daemon: %s @ %s", info.Name, info.ServerVersion)
+
+	log.Println("Creating AWS client")
+	cfg, err := awsconfig.LoadDefaultConfig(context.TODO())
+	if err != nil {
+		log.Fatalf("Unable to load AWS SDK config: %v", err)
+	}
+	// Override the AWS region with the ecrPublicRegion for ECR authentication.
+	cfg.Region = ecrPublicRegion
+
+	ecrManager := &ecrManager{client: ecrpublic.NewFromConfig(cfg)}
+	backoffSettings = backoff.NewExponentialBackOff()
+	backoffSettings.InitialInterval = 1 * time.Second
+	backoffSettings.MaxElapsedTime = 10 * time.Second
+	notifyError := func(err error, d time.Duration) {
+		log.Fatalf("%v (%s)", err, d.String())
+	}
+
+	// Build ECR registries cache.
+	if err = backoff.RetryNotify(ecrManager.buildCacheBackoff(), backoffSettings, notifyError); err != nil {
+		log.Fatalf("Could not build ECR cache: %v", err)
+	}
+
+	workersNum := runtime.NumCPU()
+
+	workerCh := make(chan []Repository, 5)
+	var wg sync.WaitGroup
+
+	for i := 0; i < workersNum; i++ {
+		go worker(&wg, workerCh, &client, ecrManager)
+	}
+
+	// Add source repo and target repo pair to worker channel.
+	for i := range config.SourceRepos {
+		wg.Add(1)
+		workerCh <- []Repository{config.SourceRepos[i], config.TargetRepos[i]}
+	}
+
+	wg.Wait()
+	log.Println("Finished mirroring repositories")
+}