diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet index e394e979436ca..703ca325056b8 100644 --- a/.drone/drone.jsonnet +++ b/.drone/drone.jsonnet @@ -32,6 +32,8 @@ local secret(name, vault_path, vault_key) = { }; local docker_username_secret = secret('docker_username', 'infra/data/ci/docker_hub', 'username'); local docker_password_secret = secret('docker_password', 'infra/data/ci/docker_hub', 'password'); +local ecr_key = secret('ecr_key', 'infra/data/ci/loki/aws-credentials', 'access_key_id'); +local ecr_secret_key = secret('ecr_secret_key', 'infra/data/ci/loki/aws-credentials', 'secret_access_key'); local pull_secret = secret('dockerconfigjson', 'secret/data/common/gcr', '.dockerconfigjson'); local github_secret = secret('github_token', 'infra/data/ci/github/grafanabot', 'pat'); @@ -73,6 +75,21 @@ local clients_docker(arch, app) = { }, }; +local lambda_promtail_ecr(app) = { + name: '%s-image' % if $.settings.dry_run then 'build-' + app else 'publish-' + app, + image: 'cstyan/ecr', + privileged: true, + settings: { + repo: 'lambda-promtail', + registry: 'public.ecr.aws/grafana', + dockerfile: 'tools/%s/Dockerfile' % app, + access_key: { from_secret: ecr_key.name }, + secret_key: { from_secret: ecr_secret_key.name }, + dry_run: false, + region: 'us-east-1', + }, +}; + local arch_image(arch, tags='') = { platform: { os: 'linux', @@ -209,6 +226,37 @@ local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) { depends_on: ['check'], }; +local lambda_promtail(tags='') = pipeline('lambda-promtail'){ + steps+: [ + { + name: 'image-tag', + image: 'alpine', + commands: [ + 'apk add --no-cache bash git', + 'git fetch origin --tags', + 'echo $(./tools/image-tag)-amd64 > .tags', + ] + if tags != '' then ['echo ",%s" >> .tags' % tags] else [], + }, + lambda_promtail_ecr('lambda-promtail') { + depends_on: ['image-tag'], + when: condition('exclude').tagMain, + settings+: { + dry_run: true, + }, + }, + ] + [ + // publish for tag or main + lambda_promtail_ecr('lambda-promtail') { + depends_on: ['image-tag'], + when: condition('include').tagMain, + settings+: { + build_args: ['TOUCH_PROTOS=1'], + }, + }, + ], + depends_on: ['check'], +}; + local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) { steps+: [ // dry run for everything that is not tag or main @@ -355,4 +403,5 @@ local manifest(apps) = pipeline('manifest') { ], }, ] + [promtail_win()] -+ [github_secret, pull_secret, docker_username_secret, docker_password_secret, deploy_configuration] ++ [lambda_promtail('latest,main')] ++ [github_secret, pull_secret, docker_username_secret, docker_password_secret, ecr_key, ecr_secret_key, deploy_configuration] diff --git a/.drone/drone.yml b/.drone/drone.yml index 3c8b8513f7001..c75e5b29264e3 100644 --- a/.drone/drone.yml +++ b/.drone/drone.yml @@ -884,6 +884,63 @@ steps: image: golang:windowsservercore-1809 name: test --- +depends_on: +- check +kind: pipeline +name: lambda-promtail +steps: +- commands: + - apk add --no-cache bash git + - git fetch origin --tags + - echo $(./tools/image-tag)-amd64 > .tags + - echo ",latest,main" >> .tags + image: alpine + name: image-tag +- depends_on: + - image-tag + image: cstyan/ecr + name: build-lambda-promtail-image + privileged: true + settings: + access_key: + from_secret: ecr_key + dockerfile: tools/lambda-promtail/Dockerfile + dry_run: true + region: us-east-1 + registry: public.ecr.aws/grafana + repo: lambda-promtail + secret_key: + from_secret: ecr_secret_key + when: + ref: + exclude: + - refs/heads/main + - refs/heads/k?? + - refs/tags/v* +- depends_on: + - image-tag + image: cstyan/ecr + name: publish-lambda-promtail-image + privileged: true + settings: + access_key: + from_secret: ecr_key + build_args: + - TOUCH_PROTOS=1 + dockerfile: tools/lambda-promtail/Dockerfile + dry_run: false + region: us-east-1 + registry: public.ecr.aws/grafana + repo: lambda-promtail + secret_key: + from_secret: ecr_secret_key + when: + ref: + include: + - refs/heads/main + - refs/heads/k?? + - refs/tags/v* +--- get: name: pat path: infra/data/ci/github/grafanabot @@ -908,6 +965,18 @@ get: kind: secret name: docker_password --- +get: + name: access_key_id + path: infra/data/ci/loki/aws-credentials +kind: secret +name: ecr_key +--- +get: + name: secret_access_key + path: infra/data/ci/loki/aws-credentials +kind: secret +name: ecr_secret_key +--- get: name: config.json path: infra/data/ci/loki/deploy @@ -915,6 +984,6 @@ kind: secret name: deploy_config --- kind: signature -hmac: ad45dfe45de92130908844a1a3cf4b829d71e057f4c9f819063a437b963232d0 +hmac: 2f519d332d5e2241a44b170461a9cf256dc8050de6d0a6caa99839a4b561aab5 ... diff --git a/Makefile b/Makefile index 4b53e0cdc9050..0b9c510017701 100644 --- a/Makefile +++ b/Makefile @@ -574,7 +574,7 @@ ifeq ($(BUILD_IN_CONTAINER),true) $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; else drone jsonnet --stream --format -V __build-image-version=$(BUILD_IMAGE_VERSION) --source .drone/drone.jsonnet --target .drone/drone.yml - drone lint .drone/drone.yml + drone lint .drone/drone.yml --trusted drone sign --save grafana/loki .drone/drone.yml || echo "You must set DRONE_SERVER and DRONE_TOKEN" endif diff --git a/tools/lambda-promtail/Dockerfile b/tools/lambda-promtail/Dockerfile index 1379df8c8d605..4eedccdb37b65 100644 --- a/tools/lambda-promtail/Dockerfile +++ b/tools/lambda-promtail/Dockerfile @@ -1,23 +1,21 @@ FROM golang:1-alpine3.12 AS build-image -WORKDIR /app +COPY tools/lambda-promtail /src/lambda-promtail +WORKDIR /src/lambda-promtail RUN go version RUN apk update && apk upgrade && \ apk add --no-cache bash git -COPY go.mod go.sum ./ RUN go mod download - -COPY lambda-promtail/main.go main.go -RUN go build -tags lambda.norpc -ldflags="-s -w" main.go +RUN go build -tags lambda.norpc -ldflags="-s -w" lambda-promtail/main.go FROM alpine:3.12 WORKDIR /app -COPY --from=build-image /app/main ./ +COPY --from=build-image /src/lambda-promtail/main ./ ENTRYPOINT ["/app/main"] \ No newline at end of file diff --git a/tools/lambda-promtail/Makefile b/tools/lambda-promtail/Makefile index 359504e2fff9d..3d10d50634b09 100644 --- a/tools/lambda-promtail/Makefile +++ b/tools/lambda-promtail/Makefile @@ -3,8 +3,5 @@ all: build docker build: GOOS=linux CGO_ENABLED=0 go build lambda-promtail/main.go -docker: - docker build --pull . - clean: rm main diff --git a/tools/lambda-promtail/README.md b/tools/lambda-promtail/README.md index 70e2156f8633f..08020859f0370 100644 --- a/tools/lambda-promtail/README.md +++ b/tools/lambda-promtail/README.md @@ -24,10 +24,9 @@ If you want to modify the lambda-promtail code you will also need: ### Building and Packaging -The provided Makefile has targets `build`, `docker`, `all`, and `clean`. +The provided Makefile has targets `build`, and `clean`. -`build`, `docker`, and `all` build the lambda-promtail as a Go static binary and use the AWS Lambda Go runtime base image to generate an image that you -can upload to your AWS ECR and use via Lambda. `clean` will remove the built Go binary. +`build` builds the lambda-promtail as a Go static binary. To build the container image properly you should run `docker build . -f tools/lambda-promtail/Dockerfile` from the root of the Loki repository,you can upload this image to your AWS ECR and use via Lambda. `clean` will remove the built Go binary. ### Packaging and deployment