diff --git a/Docker/Jenkins-CI-Worker/Dockerfile b/Docker/Jenkins-CI-Worker/Dockerfile new file mode 100644 index 000000000..e9f8dd124 --- /dev/null +++ b/Docker/Jenkins-CI-Worker/Dockerfile @@ -0,0 +1,118 @@ +FROM jenkins/jnlp-slave:4.3-1 + +USER root + +ENV DEBIAN_FRONTEND=noninteractive + +# install python +RUN set -xe && apt-get update && apt-get install -y apt-utils dnsutils python python-setuptools python-dev python-pip python3 python3-pip python3-venv build-essential zip unzip jq less vim gettext-base + +RUN set -xe && apt-get update \ + && apt-get install -y lsb-release \ + apt-transport-https \ + ca-certificates \ + curl \ + gnupg2 \ + libffi-dev \ + libssl-dev \ + libcurl4-openssl-dev \ + libncurses5-dev \ + libncursesw5-dev \ + libreadline-dev \ + libsqlite3-dev \ + libgdbm-dev \ + libdb5.3-dev \ + libbz2-dev \ + libexpat1-dev \ + liblzma-dev \ + python-virtualenv \ + lua5.3 \ + r-base \ + software-properties-common \ + sudo \ + tk-dev \ + zlib1g-dev \ + zsh \ + && ln -s /usr/bin/lua5.3 /usr/local/bin/lua + +# install google tools +RUN export CLOUD_SDK_REPO="cloud-sdk-$(lsb_release -c -s)" \ + && echo "deb https://packages.cloud.google.com/apt $CLOUD_SDK_REPO main" > /etc/apt/sources.list.d/google-cloud-sdk.list \ + && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - \ + && apt-get update \ + && apt-get install -y google-cloud-sdk \ + google-cloud-sdk-cbt \ + kubectl + +# +# install docker tools: +# * https://docs.docker.com/install/linux/docker-ce/debian/#install-docker-ce-1 +# * https://docs.docker.com/compose/install/#install-compose +# +RUN curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - \ + && add-apt-repository \ + "deb [arch=amd64] https://download.docker.com/linux/debian \ + $(lsb_release -cs) \ + stable" \ + && apt-get update \ + && apt-get install -y docker-ce \ + && curl -L "https://github.com/docker/compose/releases/download/1.23.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose \ + && chmod a+rx /usr/local/bin/docker-compose + +# install nodejs +RUN curl -sL https://deb.nodesource.com/setup_12.x | bash - +RUN apt-get update && apt-get install -y nodejs + +# add psql: https://www.postgresql.org/download/linux/debian/ +RUN DISTRO="$(lsb_release -c -s)" \ + && echo "deb http://apt.postgresql.org/pub/repos/apt/ ${DISTRO}-pgdg main" > /etc/apt/sources.list.d/pgdg.list \ + && wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add - \ + && apt-get update \ + && apt-get install -y postgresql-client-9.6 libpq-dev \ + && rm -rf /var/lib/apt/lists/* + +# Copy sh script responsible for installing Python +COPY install-python3.8.sh /root/tmp/install-python3.8.sh + +# Run the script responsible for installing Python 3.8.0 and link it to /usr/bin/python +RUN chmod +x /root/tmp/install-python3.8.sh; sync && \ + bash /root/tmp/install-python3.8.sh && \ + rm -rf /root/tmp/install-python3.8.sh && \ + unlink /usr/bin/python3 && \ + ln -s /Python-3.8.0/python /usr/bin/python3 + +RUN env +RUN which python +RUN which python3.8 + +# Fix shebang for lsb_release +RUN sed -i 's/python3/python3.5/' /usr/bin/lsb_release && \ + sed -i 's/python3/python3.5/' /usr/bin/add-apt-repository + +# install aws cli, poetry, pytest, etc. +RUN set -xe && python3.8 -m pip install awscli --upgrade && python3.8 -m pip install pytest --upgrade && python3.8 -m pip install poetry && python3.8 -m pip install PyYAML --upgrade && python3.8 -m pip install lxml --upgrade && python3.8 -m pip install yq --upgrade + +RUN curl -sSL https://raw.githubusercontent.com/python-poetry/poetry/master/get-poetry.py | python3.8 - + +# install terraform +RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.11.15/terraform_0.11.15_linux_amd64.zip \ + && unzip /tmp/terraform.zip -d /usr/local/bin && /bin/rm /tmp/terraform.zip + +RUN curl -o /tmp/terraform.zip https://releases.hashicorp.com/terraform/0.12.31/terraform_0.12.31_linux_amd64.zip \ + && unzip /tmp/terraform.zip -d /tmp && mv /tmp/terraform /usr/local/bin/terraform12 && /bin/rm /tmp/terraform.zip + +# install packer +RUN curl -o /tmp/packer.zip https://releases.hashicorp.com/packer/1.5.1/packer_1.5.1_linux_amd64.zip +RUN unzip /tmp/packer.zip -d /usr/local/bin; /bin/rm /tmp/packer.zip + +# update /etc/sudoers +RUN sed 's/^%sudo/#%sudo/' /etc/sudoers > /etc/sudoers.bak \ + && /bin/echo -e "\n%sudo ALL=(ALL:ALL) NOPASSWD:ALL\n" >> /etc/sudoers.bak \ + && cp /etc/sudoers.bak /etc/sudoers \ + && usermod -G sudo jenkins + +USER jenkins + +RUN git config --global user.email jenkins \ + && git config --global user.name jenkins + diff --git a/Docker/Jenkins-CI-Worker/README.md b/Docker/Jenkins-CI-Worker/README.md new file mode 100644 index 000000000..604c42c0b --- /dev/null +++ b/Docker/Jenkins-CI-Worker/README.md @@ -0,0 +1,2 @@ +# Overview +To be used by the `gen3-ci-worker` Jenkins worker through the JNLP connection with `jenkins-master`. diff --git a/Docker/Jenkins-CI-Worker/install-python3.8.sh b/Docker/Jenkins-CI-Worker/install-python3.8.sh new file mode 100755 index 000000000..a01d59420 --- /dev/null +++ b/Docker/Jenkins-CI-Worker/install-python3.8.sh @@ -0,0 +1,8 @@ +#!/bin/bash +wget https://www.python.org/ftp/python/3.8.0/Python-3.8.0.tar.xz +tar xf Python-3.8.0.tar.xz +rm Python-3.8.0.tar.xz +cd Python-3.8.0 +./configure +make +make altinstall diff --git a/kube/services/jenkins-ci-worker/jenkins-agent-service.yaml b/kube/services/jenkins-ci-worker/jenkins-agent-service.yaml new file mode 100644 index 000000000..7f4e58109 --- /dev/null +++ b/kube/services/jenkins-ci-worker/jenkins-agent-service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + name: jenkins-agent-service + name: jenkins-agent + namespace: default +spec: + ports: + - name: slavelistener + port: 50000 + protocol: TCP + targetPort: 50000 + selector: + app: jenkins + sessionAffinity: None + type: ClusterIP diff --git a/kube/services/jenkins-ci-worker/jenkins-worker-ci-deployment.yaml b/kube/services/jenkins-ci-worker/jenkins-worker-ci-deployment.yaml new file mode 100644 index 000000000..a02ec2ed6 --- /dev/null +++ b/kube/services/jenkins-ci-worker/jenkins-worker-ci-deployment.yaml @@ -0,0 +1,132 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: jenkins-ci-worker-deployment +spec: + selector: + # Only select pods based on the 'app' label + matchLabels: + app: jenkins-ci-worker + template: + metadata: + labels: + app: jenkins-ci-worker + # for network policy + netnolimit: "yes" + spec: + serviceAccountName: jenkins-service + securityContext: + runAsUser: 1000 + fsGroup: 1000 + initContainers: + - args: + - -c + - | + # fix permissions for /var/run/docker.sock + chmod 666 /var/run/docker.sock + echo "done" + command: + - /bin/bash + image: quay.io/cdis/awshelper:master + imagePullPolicy: Always + name: awshelper + resources: {} + securityContext: + allowPrivilegeEscalation: false + runAsUser: 0 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /var/run/docker.sock + name: dockersock + containers: + # + # See for details on running docker in a pod: + # https://estl.tech/accessing-docker-from-a-kubernetes-pod-68996709c04b + # + - name: jenkins-worker + image: "quay.io/cdis/gen3-ci-worker:master" + ports: + - containerPort: 8080 + env: + - name: JENKINS_URL + value: "https://jenkins.planx-pla.net" + - name: JENKINS_SECRET + valueFrom: + secretKeyRef: + name: jenkins-ci-worker-g3auto + key: jenkins-jnlp-agent-secret + - name: JENKINS_AGENT_NAME + value: "gen3-ci-worker" + - name: JENKINS_TUNNEL + value: "jenkins-agent:50000" + - name: AWS_DEFAULT_REGION + value: us-east-1 + - name: JAVA_OPTS + value: "-Xmx3072m" + - name: AWS_ACCESS_KEY_ID + valueFrom: + secretKeyRef: + name: jenkins-secret + key: aws_access_key_id + - name: AWS_SECRET_ACCESS_KEY + valueFrom: + secretKeyRef: + name: jenkins-secret + key: aws_secret_access_key + - name: GOOGLE_EMAIL_AUX1 + valueFrom: + secretKeyRef: + name: google-acct1 + key: email + - name: GOOGLE_PASSWORD_AUX1 + valueFrom: + secretKeyRef: + name: google-acct1 + key: password + - name: GOOGLE_EMAIL_AUX2 + valueFrom: + secretKeyRef: + name: google-acct2 + key: email + - name: GOOGLE_PASSWORD_AUX2 + valueFrom: + secretKeyRef: + name: google-acct2 + key: password + - name: GOOGLE_APP_CREDS_JSON + valueFrom: + secretKeyRef: + name: jenkins-g3auto + key: google_app_creds.json + resources: + limits: + cpu: 0.6 + memory: 2048Mi + imagePullPolicy: Always + volumeMounts: + - name: "cert-volume" + readOnly: true + mountPath: "/mnt/ssl/service.crt" + subPath: "service.crt" + - name: "cert-volume" + readOnly: true + mountPath: "/mnt/ssl/service.key" + subPath: "service.key" + - name: "ca-volume" + readOnly: true + mountPath: "/usr/local/share/ca-certificates/cdis/cdis-ca.crt" + subPath: "ca.pem" + - name: dockersock + mountPath: "/var/run/docker.sock" + imagePullPolicy: Always + volumes: + - name: cert-volume + secret: + secretName: "cert-jenkins-service" + - name: ca-volume + secret: + secretName: "service-ca" + - name: dockersock + hostPath: + path: /var/run/docker.sock diff --git a/kube/services/jenkins-ci-worker/jenkins-worker-ci-pvc.yaml b/kube/services/jenkins-ci-worker/jenkins-worker-ci-pvc.yaml new file mode 100644 index 000000000..047e4e966 --- /dev/null +++ b/kube/services/jenkins-ci-worker/jenkins-worker-ci-pvc.yaml @@ -0,0 +1,12 @@ +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: datadir-jenkins-ci + annotations: + volume.beta.kubernetes.io/storage-class: gp2 +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi diff --git a/kube/services/jenkins-worker/jenkins-worker-deployment.yaml b/kube/services/jenkins-worker/jenkins-worker-deployment.yaml index 94e822178..b1702b9e2 100644 --- a/kube/services/jenkins-worker/jenkins-worker-deployment.yaml +++ b/kube/services/jenkins-worker/jenkins-worker-deployment.yaml @@ -45,7 +45,7 @@ spec: # https://estl.tech/accessing-docker-from-a-kubernetes-pod-68996709c04b # - name: jenkins-worker - image: "registry.hub.docker.com/jenkins/jnlp-slave:4.3-1" + image: "quay.io/cdis/gen3-qa-worker:master" ports: - containerPort: 8080 env: