Setup mspai server in new infra (prebid#341)

* move config

* update helm config

* delete eks specific config

* move stored requests

* disable hooks

* set up mspai config + test plan

* disable execution plan

* add new dockerfile for mspai

* only keep one imp config for testing

* update domain

* rename account id

* add workflow

* update ECR path

* disable cross platform

* checkout submodules = true

* rename workflowe

* update image repo address

* update checkout action

* update aws account id

* update account id

* resolve comments

* setup kafka topic and update schema id

* adjust resource and k8s config

.github/workflows/mspai-ci-dev.yaml

@@ -0,0 +1,29 @@
+name: MSPAI Build dev pipeline
+on:
+  pull_request: {}
+jobs:
+  dev-build:
+    if: github.actor!= 'dependabot-preview[bot]'
+    name: build
+    runs-on:
+      group: mspai
+      labels: ci
+    timeout-minutes: 15
+    env:
+      AWS_ACCOUNT: ${{ vars.MSPAI_AWS_ACCOUNT }}
+      AWS_REGION: ${{ vars.MSPAI_AWS_REGION }}
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+      - run: pwd
+      - run: ls -la
+      - name: ECR Login
+        run: |
+          aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com
+      - name: build docker
+        id: build
+        run: |
+          printenv
+          apollo-ng

.github/workflows/mspai-ci-release.yaml

@@ -0,0 +1,78 @@
+name: Build Release Pipeline
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: "Tag name. Example: v1.0.0"
+        type: string
+        required: true
+      push_release_docker_tag:
+        description: "Push release docker tag. This will trigger ArgoCD deployment."
+        default: "yes"
+        type: choice
+        options:
+          - "yes"
+          - "no"
+
+run-name: "Release build ${{ github.event.inputs.tag }} by @${{ github.actor }}"
+
+permissions:
+  contents: write
+
+jobs:
+  build-artifact:
+    runs-on: mspai
+    name: build release artifact
+    env:
+      AWS_ACCOUNT: ${{ vars.MSPAI_AWS_ACCOUNT }}
+      AWS_REGION: ${{ vars.MSPAI_AWS_REGION }}
+      RELEASE_TAG: ${{ github.event.inputs.tag }}
+    steps:
+      - name: Validate Tag Format
+        run: |
+          if [[ ${{ github.event.inputs.tag }} =~ ^v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
+            echo "Tag format is valid. Current: ${{ github.event.inputs.tag }}"
+          else
+            echo "Invalid tag format. A valid tag format is v1.0.0"
+            exit 1
+          fi
+
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: 'true'
+
+      - name: ECR Login
+        run: |
+          aws ecr get-login-password --region us-west-2 | docker login --username AWS --password-stdin $AWS_ACCOUNT.dkr.ecr.$AWS_REGION.amazonaws.com
+
+      - name: Build and push release tag
+        if: ${{ github.event.inputs.push_release_docker_tag == 'yes' }}
+        id: build_push_release_tag
+        env:
+          RELEASE_BUILD: true
+        run: |
+          pwd
+          ls -al
+          apollo-ng
+
+      - name: Build and push tag only
+        if: ${{ github.event.inputs.push_release_docker_tag != 'yes' }}
+        id: build_push_tag
+        run: |
+          pwd
+          ls -al
+          apollo-ng
+
+      - name: Clean up
+        run: |
+          docker rmi $(docker images -f dangling=true  -q) || true
+        continue-on-error: true
+
+      - name: Push release tag
+        if: ${{ steps.build_push_tag.outcome == 'success' || steps.build_push_release_tag.outcome == 'success' }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          git tag ${{ github.event.inputs.tag }}
+          git push origin ${{ github.event.inputs.tag }}

.github/workflows/mspai-ci-tag-release.yaml

@@ -0,0 +1,35 @@
+
+name: Tag Release Pipeline
+on:
+  workflow_dispatch:
+    inputs:
+      docker-tag:
+        description: "Docker tag name. Example: release or latest"
+        required: true
+        default: "latest"
+        type: choice
+        options:
+          - release
+          - latest
+
+run-name: "Tag build ${{ github.ref }} to ${{ github.event.inputs.docker-tag }} by @${{ github.actor }}"
+
+permissions:
+  contents: write
+
+jobs:
+  build-artifact:
+    runs-on: mspai
+    name: tag image
+    env:
+      AWS_ACCOUNT: ${{ vars.MSPAI_AWS_ACCOUNT }}
+      AWS_REGION: ${{ vars.MSPAI_AWS_REGION }}
+      RELEASE_TAG: ${{ github.event.inputs.docker-tag }}
+    steps:
+      - name: Tag image
+        env:
+          RELEASE_BUILD: true
+        run: |
+          printenv
+          MANIFEST=$(aws ecr batch-get-image --repository-name ${GITHUB_REF_NAME} --image-ids imageTag=${GITHUB_REF_NAME} --output text --query 'images[].imageManifest')
+          aws ecr put-image --repository-name ${GITHUB_REF_NAME} --image-tag ${RELEASE_TAG} --image-manifest "$MANIFEST"

.gitignore

@@ -1,4 +1,5 @@
 pbs.yaml
+mspai_pbs.yaml
 stored_requests/*
 vendor/*
 # just for local testing purpose

Dockerfile.mspai

@@ -0,0 +1,33 @@
+FROM 590183745426.dkr.ecr.us-west-2.amazonaws.com/particlemedia/msp:base-ci-0.0.3 AS build
+
+# Build App
+RUN mkdir -p /app/msp/
+WORKDIR /app/msp/
+COPY ./ ./
+ENV CGO_ENABLED 1
+ENV GOPRIVATE "github.com/ParticleMedia/ab-go-sdk"
+RUN go install github.com/actgardner/gogen-avro/v10/cmd/...@latest
+ENV PATH=/root/go/bin:$PATH
+RUN make avro
+RUN make build
+# RUN go build -mod=vendor -ldflags "-X github.com/prebid/prebid-server/version.Ver=`git describe --tags | sed 's/^v//'` -X github.com/prebid/prebid-server/version.Rev=`git rev-parse HEAD`" .
+
+FROM ubuntu:20.04 AS release
+LABEL maintainer="songyan.hou@newsbreak.com" 
+WORKDIR /usr/local/bin/
+COPY --from=build /app/msp/submodules/prebid-server/prebid-server .
+RUN chmod a+xr prebid-server
+COPY --from=build /app/msp/submodules/prebid-server/static static/
+COPY --from=build /app/msp/submodules/prebid-server/stored_requests/data stored_requests/data
+RUN chmod -R a+r static/ stored_requests/data
+COPY --from=build /app/msp/targets/* ./targets/
+
+RUN apt-get update && \
+    apt-get install --no-install-recommends -y ca-certificates mtr && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+RUN adduser msp_user
+USER msp_user
+EXPOSE 8000
+EXPOSE 6060
+ENTRYPOINT ["/usr/local/bin/prebid-server"]
+CMD ["-v", "1", "-logtostderr"]

Makefile

@@ -66,4 +66,8 @@ link_config:
 	-rm $(PREBID_SERVER_DIR)/pbs.yaml
 	@cd $(PREBID_SERVER_DIR); ln -s ../../pbs.yaml pbs.yaml
 
+link_config_mspai:
+	-rm $(PREBID_SERVER_DIR)/pbs.yaml
+	@cd $(PREBID_SERVER_DIR); ln -s ../../mspai_pbs.yaml pbs.yaml
+
 .PHONY: deps test build image

apollo.yaml

@@ -0,0 +1,3 @@
+msp:
+  dockerfile_path: ./Dockerfile.mspai
+  docker_platform: linux/amd64

helm/mspai/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

helm/mspai/Chart.yaml

@@ -0,0 +1,21 @@
+apiVersion: v2
+name: msp
+description: A Helm chart for Kubernetes
+
+# A chart can be either an 'application' or a 'library' chart.
+#
+# Application charts are a collection of templates that can be packaged into versioned archives
+# to be deployed.
+#
+# Library charts provide useful utilities or functions for the chart developer. They're included as
+# a dependency of application charts to inject those utilities and functions into the rendering
+# pipeline. Library charts do not define any templates and therefore cannot be deployed.
+type: application
+
+# This is the chart version. This version number should be incremented each time you make changes
+# to the chart and its templates, including the app version.
+version: 0.1.0
+
+# This is the version number of the application being deployed. This version number should be
+# incremented each time you make changes to the application.
+appVersion: 1.16.0

helm/mspai/READMD.md

@@ -0,0 +1,27 @@
+## List existing release
+
+```
+helm ls -n monetization
+```
+
+## Upgrade or install application
+
+```
+helm upgrade -i msp helm/msp \
+    -f helm/msp/values.yaml \
+    -f helm/msp/values-stage.yaml \
+    --namespace monetization \
+    --dry-run --debug
+
+helm upgrade -i msp helm/msp \
+    -f helm/msp/values.yaml \
+    -f helm/msp/values-prod.yaml \
+    --namespace monetization \
+    --dry-run --debug
+```
+
+## Delete existing deployment
+
+```
+helm delete msp -n monetization
+```

helm/mspai/templates/NOTES.txt

@@ -0,0 +1,21 @@
+1. Get the application URL by running these commands:
+{{- if .Values.ingress.enabled }}
+{{- range $host := .Values.ingress.hosts }}
+  {{- range .paths }}
+  http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ . }}
+  {{- end }}
+{{- end }}
+{{- else if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "app-server.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "app-server.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "app-server.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "app-server.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:80
+{{- end }}

helm/mspai/templates/_helpers.tpl

@@ -0,0 +1,63 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "app-server.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "app-server.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "app-server.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Common labels
+*/}}
+{{- define "app-server.labels" -}}
+helm.sh/chart: {{ include "app-server.chart" . }}
+{{ include "app-server.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end -}}
+
+{{/*
+Selector labels
+*/}}
+{{- define "app-server.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "app-server.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end -}}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "app-server.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create -}}
+    {{ default (include "app-server.fullname" .) .Values.serviceAccount.name }}
+{{- else -}}
+    {{ default "default" .Values.serviceAccount.name }}
+{{- end -}}
+{{- end -}}

helm/mspai/templates/configmap.yaml

@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  labels:
+    {{- include "app-server.labels" . | nindent 4 }}
+  name: {{ include "app-server.fullname" . }}
+  namespace: {{ .Release.Namespace }}
+data:
+  {{ .Values.server.config.key }}: |
+{{ .Values.server.config.value | toString | indent 4 }}