diff --git a/.github/workflows/CD.yml b/.github/workflows/CD.yml index 33c654d178..a8f2e91029 100644 --- a/.github/workflows/CD.yml +++ b/.github/workflows/CD.yml @@ -343,6 +343,7 @@ jobs: docker tag $IMAGE_NAME public.ecr.aws/$ECR_REPO:latest docker push public.ecr.aws/$ECR_REPO:$TAG docker push public.ecr.aws/$ECR_REPO:latest + go run tools/release/adot-operator-images-mirror - name: Login Dockerhub uses: docker/login-action@v1 diff --git a/THIRD-PARTY-LICENSES.txt b/THIRD-PARTY-LICENSES.txt index 4ff05712af..d120167911 100644 --- a/THIRD-PARTY-LICENSES.txt +++ b/THIRD-PARTY-LICENSES.txt @@ -849,4 +849,36 @@ Exhibit B - "Incompatible With Secondary Licenses" Notice --------------------------------------------------------- This Source Code Form is "Incompatible With Secondary Licenses", as - defined by the Mozilla Public License, v. 2.0. \ No newline at end of file + defined by the Mozilla Public License, v. 2.0. + +------ + +BSD 3-Clause License + +Copyright (c) 2017, SeatGeek +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/go.mod b/go.mod index 8f858cc930..24ac484a94 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,11 @@ module github.com/aws-observability/aws-otel-collector go 1.14 require ( + github.com/aws/aws-sdk-go-v2/config v1.5.0 + github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.4.1 + github.com/cenkalti/backoff v2.2.1+incompatible github.com/crossdock/crossdock-go v0.0.0-20160816171116-049aabb0122b + github.com/fsouza/go-dockerclient v1.7.3 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsemfexporter v0.30.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsprometheusremotewriteexporter v0.30.0 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/awsxrayexporter v0.30.0 @@ -28,6 +32,7 @@ require ( go.uber.org/zap v1.18.1 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 gopkg.in/natefinch/lumberjack.v2 v2.0.0 + gopkg.in/yaml.v2 v2.4.0 ) replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray => github.com/open-telemetry/opentelemetry-collector-contrib/internal/aws/xray v0.30.0 diff --git a/go.sum b/go.sum index 443351f706..4ec3d53b57 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,5 @@ 4d63.com/gochecknoglobals v0.0.0-20201008074935-acfc0b28355a/go.mod h1:wfdC5ZjKSPr7CybKEcgJhUOgeAQW1+7WcyK8OvUilfo= +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -125,8 +126,12 @@ github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF0 github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.5.0 h1:Elr9Wn+sGKPlkaBvwu4mTrxtmOp3F3yV9qhaHbXGjwU= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/hcsshim v0.8.14 h1:lbPVK25c1cu5xTLITwpUcxoA9vKrKErASPYygvouJns= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= @@ -203,8 +208,28 @@ github.com/aws/aws-sdk-go v1.39.5 h1:yoJEE1NJxbpZ3CtPxvOSFJ9ByxiXmBTKk8J+XU5ldtg github.com/aws/aws-sdk-go v1.39.5/go.mod h1:585smgzpB/KqRA+K3y/NL/oYRqQvpNJYvLm+LY1U59Q= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.7.0/go.mod h1:tb9wi5s61kTDA5qCkcDbt3KRVV74GGslQkl/DRdX/P4= +github.com/aws/aws-sdk-go-v2 v1.7.1 h1:TswSc7KNqZ/K1Ijt3IkpXk/2+62vi3Q82Yrr5wSbRBQ= +github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= +github.com/aws/aws-sdk-go-v2/config v1.5.0 h1:tRQcWXVmO7wC+ApwYc2LiYKfIBoIrdzcJ+7HIh6AlR0= +github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA= +github.com/aws/aws-sdk-go-v2/credentials v1.3.1 h1:fFeqL5+9kwFKsCb2oci5yAIDsWYqn/Nga8oQ5bIasI8= +github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0 h1:s4vtv3Mv1CisI3qm2HGHi1Ls9ZtbCOEqeQn6oz7fTyU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1 h1:SDLwr1NKyowP7uqxuLNdvFZhjnoVWxNv456zAp+ZFjU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg= github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.5.0/go.mod h1:acH3+MQoiMzozT/ivU+DbRg7Ooo2298RdRaWcOv+4vM= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.4.1 h1:zoktIoJ+S7mJpABtSYzcVCFosRK1zehO78Lc86AbOQk= +github.com/aws/aws-sdk-go-v2/service/ecrpublic v1.4.1/go.mod h1:eD5Eo4drVP2FLTw0G+SMIPWNWvQRGGTtIZR2XeAagoA= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1 h1:VJe/XEhrfyfBLupcGg1BfUSK2VMZNdbDcZQ49jnp+h0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.1 h1:H2ZLWHUbbeYtghuqCY5s/7tbBM99PAwCioRJF8QvV/U= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.1/go.mod h1:J3A3RGUvuCZjvSuZEcOpHDnzZP/sKbhDWV2T1EOzFIM= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.0 h1:Y9r6mrzOyAYz4qKaluSH19zqH1236il/nGbsPKOUT0s= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg= github.com/aws/smithy-go v1.5.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/aws/smithy-go v1.6.0 h1:T6puApfBcYiTIsaI+SYWqanjMt5pc3aoyyDrI+0YH54= +github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/beeker1121/goque v2.0.1+incompatible h1:5nJHPMqQLxUvGFc8m/NW2QzxKyc0zICmqs/JUsmEjwE= github.com/beeker1121/goque v2.0.1+incompatible/go.mod h1:L6dOWBhDOnxUVQsb0wkLve0VCnt2xJW/MI8pdRX4ANw= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= @@ -256,6 +281,8 @@ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5P github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575 h1:kHaBemcxl8o/pQ5VM1c8PVE1PubbNx3mjUr09OqWGCs= github.com/cihub/seelog v0.0.0-20170130134532-f561c5e57575/go.mod h1:9d6lWj8KzO/fd/NrVaLscBKmPigpZpn5YawRPw+e3Yo= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.5.0 h1:E1KshmrMEtkMP2UjlWzfmUV1owWY+BnbL5FxxuatnrU= github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2 h1:iHsfF/t4aW4heW2YKfeHrVPGdtYTL4C4KocpM8KTSnI= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= @@ -269,13 +296,24 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v1.0.2 h1:Pi6D+aZXM+oUw1czuKgH5IJ+y0jhYcwBJfx5/Ghn9dE= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.4 h1:rtRG4N6Ct7GNssATwgpvMGfnjnwfjnu/Zs9W3Ikzq+M= github.com/containerd/containerd v1.4.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e h1:6JKvHHt396/qabvMhnhUZvWaHZzfVfldxE60TK8YLhg= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v1.0.1 h1:PvuK4E3D5S5q6IqsPDCy928FhP0LUIGcmZ/Yhgp5Djw= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -289,6 +327,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= @@ -408,6 +447,8 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsouza/go-dockerclient v1.7.3 h1:i6iMcktl688vsKUEExA6gU1UjPgIvmGtJeQ0mbuFqZo= +github.com/fsouza/go-dockerclient v1.7.3/go.mod h1:8xfZB8o9SptLNJ13VoV5pMiRbZGWkU/Omu5VOu/KC9Y= github.com/fzipp/gocyclo v0.3.1/go.mod h1:DJHO6AUmbdqj2ET4Z9iArSuwWgYDRryYt2wASxc7x3E= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= @@ -586,6 +627,7 @@ github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY9 github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocql/gocql v0.0.0-20200228163523-cd4b606dd2fb/go.mod h1:DL0ekTmBSTdlNF25Orwt/JMzqIq3EJ4MVa/J/uK64OY= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= @@ -1084,6 +1126,9 @@ github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zx github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mjibson/esc v0.2.0/go.mod h1:9Hw9gxxfHulMF5OJKCyhYD7PzlSdhzXyaGEBRPH1OPs= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mount v0.2.0 h1:WhCW5B355jtxndN5ovugJlMFJawbUODuW8fSnEH6SSM= +github.com/moby/sys/mount v0.2.0/go.mod h1:aAivFE2LB3W4bACsUXChRHQ0qKWsetY4Y9V7sxOougM= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= @@ -1217,10 +1262,16 @@ github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiv github.com/open-telemetry/opentelemetry-collector-contrib/receiver/awsxrayreceiver v0.30.0/go.mod h1:Etcp8+MQcq/eqGiqlkCu0S6+TodtsC3GEwayl7VmEck= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver v0.30.0 h1:4ltYYbh74f6lo+tjFKur7MQdbSl6LUXyH36NBrD04X4= github.com/open-telemetry/opentelemetry-collector-contrib/receiver/statsdreceiver v0.30.0/go.mod h1:E16gOM7CZnhI2RHNAHkEgZx5SMTzQ6cHBuMph8VjyvU= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc95 h1:RMuWVfY3E1ILlVsC3RhIq38n4sJtlOFwU9gfFZSqrd0= +github.com/opencontainers/runc v1.0.0-rc95/go.mod h1:z+bZxa/+Tz/FmYVWkhUajJdzFeOqjc5vrqskhVyHGUM= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runc v1.0.0-rc95/go.mod h1:z+bZxa/+Tz/FmYVWkhUajJdzFeOqjc5vrqskhVyHGUM= github.com/opencontainers/runc v1.0.1 h1:G18PGckGdAm3yVQRWDVQ1rLSLntiniKJ0cNRT2Tm5gs= github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= @@ -1334,6 +1385,7 @@ github.com/prometheus/common v0.28.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1461,6 +1513,7 @@ github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v0.0.7/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= @@ -1472,6 +1525,7 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= @@ -1548,6 +1602,7 @@ github.com/ultraware/funlen v0.0.3/go.mod h1:Dp4UiAus7Wdb9KUZsYWZEWiRzGuM2kXM1lP github.com/ultraware/whitespace v0.0.4/go.mod h1:aVMh/gQve5Maj9hQ/hg+F75lr/X5A89uZnzAmWSineA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/uudashr/gocognit v1.0.1/go.mod h1:j44Ayx2KW4+oB6SWMv8KsmHzZrOInQav7D3cQMJ5JUM= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasthttp v1.16.0/go.mod h1:YOKImeEosDdBPnxc0gy7INqi3m1zK6A+xl6TwOBhHCA= @@ -1887,6 +1942,7 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1921,6 +1977,7 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201024232916-9f70ab9862d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1929,6 +1986,7 @@ golang.org/x/sys v0.0.0-20201214210602-f9fddec55a1e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210216224549-f992740a1bac/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1949,6 +2007,7 @@ golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210611083646-a4fc73990273/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1 h1:SrN+KX8Art/Sf4HNj6Zcz06G7VEz+7w9tdXTPOZ7+l4= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -2297,6 +2356,8 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/zorkian/go-datadog-api.v2 v2.30.0 h1:umQdVO0Ytx+kYadhuJNjFtDgIsIEBnKrOTvNuu8ClKI= gopkg.in/zorkian/go-datadog-api.v2 v2.30.0/go.mod h1:kx0CSMRpzEZfx/nFH62GLU4stZjparh/BRpM89t4XCQ= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/tools/release/adot-operator-images-mirror/config.yaml b/tools/release/adot-operator-images-mirror/config.yaml new file mode 100644 index 0000000000..26728c7378 --- /dev/null +++ b/tools/release/adot-operator-images-mirror/config.yaml @@ -0,0 +1,14 @@ +## The source repositories and target repositories are one-to-one correspondence. +sourceRepos: + - registry: opentelemetry + name: opentelemetry-operator + host: quay.io + - registry: kubebuilder + name: kube-rbac-proxy + host: gcr.io + +targetRepos: + - registry: public.ecr.aws/aws-observability + name: adot-operator + - registry: public.ecr.aws/aws-observability + name: mirror-kube-rbac-proxy diff --git a/tools/release/adot-operator-images-mirror/docker_auth.go b/tools/release/adot-operator-images-mirror/docker_auth.go new file mode 100644 index 0000000000..96a83862a9 --- /dev/null +++ b/tools/release/adot-operator-images-mirror/docker_auth.go @@ -0,0 +1,22 @@ +package main + +import ( + "fmt" + "log" + + docker "github.com/fsouza/go-dockerclient" +) + +func getDockerCredentials(registry string) (*docker.AuthConfiguration, error) { + authOptions, err := docker.NewAuthConfigurationsFromDockerCfg() + if err != nil { + log.Fatal(err) + } + + creds, ok := authOptions.Configs[registry] + if !ok { + return nil, fmt.Errorf("no auth found for %s", registry) + } + + return &creds, nil +} diff --git a/tools/release/adot-operator-images-mirror/ecr_manager.go b/tools/release/adot-operator-images-mirror/ecr_manager.go new file mode 100644 index 0000000000..71c395b151 --- /dev/null +++ b/tools/release/adot-operator-images-mirror/ecr_manager.go @@ -0,0 +1,94 @@ +package main + +import ( + "context" + "fmt" + "log" + + "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/cenkalti/backoff" +) + +const ( + operatorRepo = "adot-operator" + rbacProxyRepo = "mirror-kube-rbac-proxy" +) + +type ecrManager struct { + client *ecrpublic.Client + repositories map[string]bool +} + +func (e *ecrManager) exists(name string) bool { + if _, ok := e.repositories[name]; ok { + return true + } + + return false +} + +func (e *ecrManager) ensure(name string) error { + if e.exists(name) { + return nil + } + + return e.create(name) +} + +func (e *ecrManager) create(name string) error { + if name != operatorRepo && name != rbacProxyRepo { + return fmt.Errorf("wrong repository name: %s", name) + } + + _, err := e.client.CreateRepository(context.TODO(), &ecrpublic.CreateRepositoryInput{ + RepositoryName: &name, + }) + if err != nil { + return err + } + + e.repositories[name] = true + return nil +} + +func (e *ecrManager) buildCache(nextToken *string) error { + if nextToken == nil { + log.Println("Loading the list of ECR repositories") + } + + resp, err := e.client.DescribeRepositories(context.TODO(), &ecrpublic.DescribeRepositoriesInput{ + NextToken: nextToken, + }) + if err != nil { + return err + } + + if e.repositories == nil { + e.repositories = make(map[string]bool) + } + + for _, repo := range resp.Repositories { + e.repositories[*repo.RepositoryName] = true + } + + // keep paging as long as there is a token for the next page + if resp.NextToken != nil { + err := e.buildCache(resp.NextToken) + if err != nil { + return err + } + } + + // no next token means we hit the last page + if nextToken == nil { + log.Println("Done loading ECR repositories") + } + + return nil +} + +func (e *ecrManager) buildCacheBackoff() backoff.Operation { + return func() error { + return e.buildCache(nil) + } +} diff --git a/tools/release/adot-operator-images-mirror/main.go b/tools/release/adot-operator-images-mirror/main.go new file mode 100644 index 0000000000..390764fe27 --- /dev/null +++ b/tools/release/adot-operator-images-mirror/main.go @@ -0,0 +1,124 @@ +package main + +import ( + "context" + "io/ioutil" + "log" + "runtime" + "sync" + "time" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/service/ecrpublic" + "github.com/cenkalti/backoff" + docker "github.com/fsouza/go-dockerclient" + "gopkg.in/yaml.v2" +) + +const ecrPublicRegion = "us-east-1" + +var ( + config Config + backoffSettings *backoff.ExponentialBackOff +) + +type Config struct { + SourceRepos []Repository `yaml:"sourceRepos"` + TargetRepos []Repository `yaml:"targetRepos"` +} + +type Repository struct { + Registry string `yaml:"registry"` + Name string `yaml:"name"` + Host string `yaml:"host"` +} + +// Create the Docker client which will be used to pull, retag and push images. +func createDockerClient() (*docker.Client, error) { + client, err := docker.NewClientFromEnv() + return client, err +} + +// worker is the mirror working function. +func worker(wg *sync.WaitGroup, workerCh chan []Repository, dc *DockerClient, ecrm *ecrManager) { + for repos := range workerCh { + log.Printf("Starting to mirror repo %s/%s/%s to repo %s/%s", repos[0].Host, repos[0].Registry, repos[0].Name, repos[1].Registry, repos[1].Name) + + m := mirror{ + dockerClient: dc, + ecrManager: ecrm, + } + if err := m.setup(repos); err != nil { + log.Printf("Failed to setup mirror for repo %s/%s/%s: %v", repos[0].Host, repos[0].Registry, repos[0].Name, err) + wg.Done() + continue + } + + m.work() + log.Printf("From repo %s/%s/%s to repo %s/%s mirror completed", repos[0].Host, repos[0].Registry, repos[0].Name, repos[1].Registry, repos[1].Name) + wg.Done() + } +} + +func main() { + content, err := ioutil.ReadFile("tools/release/adot-operator-images-mirror/config.yaml") + if err != nil { + log.Fatalf("Could not read config file: %v", err) + } + + if err = yaml.Unmarshal(content, &config); err != nil { + log.Fatalf("Could not parse config file: %v", err) + } + + log.Println("Creating Docker client") + var client DockerClient + client, err = createDockerClient() + if err != nil { + log.Fatalf("Could not create Docker client: %v", err) + } + + info, err := client.Info() + if err != nil { + log.Fatalf("Could not get Docker info: %v", err) + } + log.Printf("Connected to Docker daemon: %s @ %s", info.Name, info.ServerVersion) + + log.Println("Creating AWS client") + cfg, err := awsconfig.LoadDefaultConfig(context.TODO()) + if err != nil { + log.Fatalf("Unable to load AWS SDK config: %v", err) + } + // Override the AWS region with the ecrPublicRegion for ECR authentication. + cfg.Region = ecrPublicRegion + + ecrManager := &ecrManager{client: ecrpublic.NewFromConfig(cfg)} + backoffSettings = backoff.NewExponentialBackOff() + backoffSettings.InitialInterval = 1 * time.Second + backoffSettings.MaxElapsedTime = 10 * time.Second + notifyError := func(err error, d time.Duration) { + log.Fatalf("%v (%s)", err, d.String()) + } + + // Build ECR registries cache. + if err = backoff.RetryNotify(ecrManager.buildCacheBackoff(), backoffSettings, notifyError); err != nil { + log.Fatalf("Could not build ECR cache: %v", err) + } + + workersNum := runtime.NumCPU() + + workerCh := make(chan []Repository, 5) + var wg sync.WaitGroup + + for i := 0; i < workersNum; i++ { + go worker(&wg, workerCh, &client, ecrManager) + } + + // Add source repo and target repo pair to worker channel. + for i := range config.SourceRepos { + wg.Add(1) + workerCh <- []Repository{config.SourceRepos[i], config.TargetRepos[i]} + } + + wg.Wait() + log.Println("Finished mirroring repositories") +} diff --git a/tools/release/adot-operator-images-mirror/mirror.go b/tools/release/adot-operator-images-mirror/mirror.go new file mode 100644 index 0000000000..f669d3889b --- /dev/null +++ b/tools/release/adot-operator-images-mirror/mirror.go @@ -0,0 +1,241 @@ +package main + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "strconv" + "time" + + "github.com/cenkalti/backoff" + docker "github.com/fsouza/go-dockerclient" +) + +const ( + ecrPublic = "public.ecr.aws" + quay = "quay.io" + gcr = "gcr.io" + defaultSleepDuration = 60 * time.Second +) + +var ( + httpClient = &http.Client{Timeout: 10 * time.Second} +) + +// QuayTagsResponse contains the tags information of the HTTP get response from quay.io. +type QuayTagsResponse struct { + Tags []RepositoryTag `json:"tags"` +} + +// GCRTagsResponse contains the tags information of the HTTP get response from gcr.io. +type GCRTagsResponse struct { + Tags []string `json:"tags"` +} + +// RepositoryTag holds the individual tag for the requested repository. +type RepositoryTag struct { + Name string `json:"name"` +} + +type DockerClient interface { + Info() (*docker.DockerInfo, error) + TagImage(string, docker.TagImageOptions) error + PullImage(docker.PullImageOptions, docker.AuthConfiguration) error + PushImage(docker.PushImageOptions, docker.AuthConfiguration) error + RemoveImage(string) error +} + +type mirror struct { + dockerClient *DockerClient // docker client used to pull, tag and push images + ecrManager *ecrManager // ECR manager, used to ensure the ECR repository exist + sourceRepo Repository // source repo to mirror from + targetRepo Repository // target repo to mirror to + remoteTags []RepositoryTag // list of remote repository tags (post filtering) +} + +func (m *mirror) setup(repos []Repository) (err error) { + m.sourceRepo = repos[0] + m.targetRepo = repos[1] + + // Fetch remote tags from source repository. + m.getRemoteTags() + + // Filter the tags we got. + m.filterTags() + + return nil +} + +func (m *mirror) pullImage(tag string) error { + pullOptions := docker.PullImageOptions{ + Repository: m.sourceRepositoryFullName(), + Tag: tag, + InactivityTimeout: 1 * time.Minute, + } + + return (*m.dockerClient).PullImage(pullOptions, docker.AuthConfiguration{}) +} + +func (m *mirror) tagImage(tag string) error { + tagOptions := docker.TagImageOptions{ + Repo: m.targetRepositoryName(), + Tag: tag, + Force: true, + } + + return (*m.dockerClient).TagImage(m.sourceRepositoryFullName()+":"+tag, tagOptions) +} + +func (m *mirror) pushImage(tag string) error { + pushOptions := docker.PushImageOptions{ + Name: m.targetRepositoryName(), + Registry: m.targetRepo.Registry, + Tag: tag, + InactivityTimeout: 1 * time.Minute, + } + + // Get ecrpublic credentials. + creds, err := getDockerCredentials(ecrPublic) + if err != nil { + return err + } + + return (*m.dockerClient).PushImage(pushOptions, *creds) +} + +func (m *mirror) work() { + if err := m.ecrManager.ensure(m.targetRepo.Name); err != nil { + log.Fatalf("Failed to create ECR repo %s: %v", m.targetRepo.Name, err) + } + + for _, tag := range m.remoteTags { + if err := m.pullImage(tag.Name); err != nil { + log.Printf("Failed to pull docker image %s:%s: %v", m.sourceRepositoryFullName(), tag.Name, err) + continue + } + + if err := m.tagImage(tag.Name); err != nil { + log.Printf("Failed to retag docker image %s:%s: %v", m.sourceRepositoryFullName(), tag.Name, err) + continue + } + + if err := m.pushImage(tag.Name); err != nil { + log.Printf("Failed to push retagged image %s:%s: %v", m.targetRepositoryName(), tag.Name, err) + continue + } + } +} + +func (m *mirror) getRemoteTags() { + var url string + switch m.sourceRepo.Host { + case quay: + url = fmt.Sprintf("https://quay.io/api/v1/repository/%s/tag", m.sourceRepositoryName()) + case gcr: + url = fmt.Sprintf("https://gcr.io/v2/%s/tags/list", m.sourceRepositoryName()) + } + + if err := backoff.Retry(m.getTagResponseBackoff(url), backoffSettings); err != nil { + log.Fatalf("Could not fetch tag information for %s: %v", url, err) + } + + log.Printf("Finished scraping remote tags from %s", m.sourceRepositoryFullName()) +} + +func (m *mirror) filterTags() { + tmp := make([]RepositoryTag, 0) + + for _, remoteTag := range m.remoteTags { + // We only keep the tags starting with "v" or with the name "latest". + if string(remoteTag.Name[0]) == "v" || remoteTag.Name == "latest" { + tmp = append(tmp, remoteTag) + } + } + + m.remoteTags = tmp +} + +func (m *mirror) sourceRepositoryName() string { + return fmt.Sprintf("%s/%s", m.sourceRepo.Registry, m.sourceRepo.Name) +} + +func (m *mirror) sourceRepositoryFullName() string { + return fmt.Sprintf("%s/%s/%s", m.sourceRepo.Host, m.sourceRepo.Registry, m.sourceRepo.Name) +} + +func (m *mirror) targetRepositoryName() string { + return fmt.Sprintf("%s/%s", m.targetRepo.Registry, m.targetRepo.Name) +} + +func (m *mirror) getTagResponse(url string) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return err + } + + res, err := httpClient.Do(req) + if err != nil { + log.Printf("Failed to get %s, retrying", url) + return err + } else if res.StatusCode == 429 { + sleepTime := getSleepTime(res.Header.Get("X-RateLimit-Reset"), time.Now()) + log.Printf("Rate limited on %s, sleeping for %s", url, sleepTime) + time.Sleep(sleepTime) + return fmt.Errorf("encouter error: too many requests") + } else if res.StatusCode < 200 || res.StatusCode >= 300 { + return fmt.Errorf("get %s failed with %d, retrying", url, res.StatusCode) + } else { + defer res.Body.Close() + + // Decode the response and add the tags to remoteTags field of mirror struct. + var allTags []RepositoryTag + dc := json.NewDecoder(res.Body) + + switch m.sourceRepo.Host { + case quay: + var tags QuayTagsResponse + if err := dc.Decode(&tags); err != nil { + return err + } + allTags = append(allTags, tags.Tags...) + case gcr: + var tags GCRTagsResponse + if err := dc.Decode(&tags); err != nil { + return err + } + for _, tag := range tags.Tags { + allTags = append(allTags, RepositoryTag{ + Name: tag, + }) + } + } + + m.remoteTags = allTags + + return nil + } +} + +func (m *mirror) getTagResponseBackoff(url string) backoff.Operation { + return func() error { + return m.getTagResponse(url) + } +} + +func getSleepTime(rateLimitReset string, now time.Time) time.Duration { + rateLimitResetInt, err := strconv.ParseInt(rateLimitReset, 10, 64) + + if err != nil { + return defaultSleepDuration + } + + sleepTime := time.Unix(rateLimitResetInt, 0) + calculatedSleepTime := sleepTime.Sub(now) + + if calculatedSleepTime < (0 * time.Second) { + return 0 * time.Second + } + + return calculatedSleepTime +}