Merge pull request #2019 from marseel/speed_up_perfdash_preloading

Add cache for list of files in bucket

perfdash/Dockerfile

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-ARG GOLANG_VERSION=1.13
+ARG GOLANG_VERSION=1.17
 FROM golang:${GOLANG_VERSION} as builder
 
 WORKDIR /workspace

perfdash/Makefile

@@ -1,7 +1,7 @@
 all: push
 
 # See deployment.yaml for the version currently running-- bump this ahead before rebuilding!
-TAG = 2.38
+TAG = 2.39
 
 REPO = gcr.io/k8s-staging-perf-tests
 

perfdash/gcs_metrics_bucket.go

@@ -92,9 +92,15 @@ func (b *GCSMetricsBucket) ListFilesInBuild(job string, buildNumber int, prefix
 	var files []string
 	ctx := context.Background()
 	jobPrefix := joinStringsAndInts(b.logPath, job, buildNumber, prefix)
-	it := b.bucket.Objects(ctx, &storage.Query{
+	query := &storage.Query{
 		Prefix: jobPrefix,
-	})
+	}
+	err := query.SetAttrSelection([]string{"Name"})
+	if err != nil {
+		return nil, err
+	}
+
+	it := b.bucket.Objects(ctx, query)
 	for {
 		attrs, err := it.Next()
 		if err == iterator.Done {
@@ -109,6 +115,10 @@ func (b *GCSMetricsBucket) ListFilesInBuild(job string, buildNumber int, prefix
 	return files, nil
 }
 
+func (b *GCSMetricsBucket) GetFilePrefix(job string, buildNumber int, prefix string) string {
+	return joinStringsAndInts(b.logPath, job, buildNumber, prefix)
+}
+
 // ReadFile reads the file contents from the GCS bucket.
 func (b *GCSMetricsBucket) ReadFile(job string, buildNumber int, path string) ([]byte, error) {
 	ctx := context.Background()

perfdash/go.mod

@@ -1,21 +1,63 @@
 module k8s.io/perf-tests/perfdash
 
-go 1.13
+go 1.17
 
 require (
-	cloud.google.com/go v0.34.0
+	cloud.google.com/go v0.100.2
+	cloud.google.com/go/storage v1.10.0
 	github.com/aws/aws-sdk-go v1.34.4
-	github.com/ghodss/yaml v0.0.0-20180820084758-c7ce16629ff4
-	github.com/google/martian v2.1.0+incompatible // indirect
-	github.com/googleapis/gax-go v1.0.3 // indirect
+	github.com/ghodss/yaml v1.0.0
 	github.com/spf13/pflag v1.0.1
-	go.opencensus.io v0.22.3 // indirect
-	google.golang.org/api v0.0.0-20181220000619-583d854617af
+	google.golang.org/api v0.70.0
 	gopkg.in/yaml.v2 v2.2.8
 	k8s.io/klog v0.3.1
 	k8s.io/kubernetes v1.15.0
 )
 
+require (
+	cloud.google.com/go/compute v1.3.0 // indirect
+	cloud.google.com/go/iam v0.3.0 // indirect
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415 // indirect
+	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect
+	github.com/google/martian v2.1.0+incompatible // indirect
+	github.com/googleapis/gax-go v1.0.3 // indirect
+	github.com/googleapis/gax-go/v2 v2.1.1 // indirect
+	github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d // indirect
+	github.com/jmespath/go-jmespath v0.3.0 // indirect
+	github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
+	github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 // indirect
+	go.opencensus.io v0.23.0 // indirect
+	golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 // indirect
+	golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect
+	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
+	golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd // indirect
+	golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 // indirect
+	golang.org/x/sys v0.0.0-20220209214540-3681064d5158 // indirect
+	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
+	golang.org/x/tools v0.1.5 // indirect
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf // indirect
+	google.golang.org/grpc v1.44.0 // indirect
+	google.golang.org/protobuf v1.27.1 // indirect
+	gopkg.in/inf.v0 v0.9.0 // indirect
+	honnef.co/go/tools v0.0.1-2020.1.4 // indirect
+	k8s.io/api v0.0.0 // indirect
+	k8s.io/apimachinery v0.0.0 // indirect
+	k8s.io/client-go v0.0.0 // indirect
+	k8s.io/utils v0.0.0-20190221042446-c2654d5206da // indirect
+	sigs.k8s.io/yaml v1.1.0 // indirect
+)
+
 replace (
 	k8s.io/api => k8s.io/api v0.0.0-20200131232428-e3a917c59b04
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.0.0-20200131234631-01db4fb12417